boto and load autoscaling groups

In boto, get_all_groups has a max return of 100. to get past that, you use tokens:

asconn = boto.ec2.autoscale.connect_to_region(self.region_name)
# This will load all groups.
# Source: http://stackoverflow.com/questions/29317526/how-can-i-retrieve-more-than-50-autoscaling-groups-via-python-boto
all_groups = []
rs = asconn.get_all_groups()
all_groups.extend(rs)
while rs.next_token:
    rs = asconn.get_all_groups(next_token=rs.next_token)
    all_groups.extend(rs)
for asg in all_groups:
    print self.stack_name + " = " + asg.name

python commit and push to git in Jenkins

I’ve looked all over the internet and can’t find anything regarding pushing to a remote repo via python…. particularly in Jenkins where your upstream is not set.

So I finally found the only helpful link and it was saying: don’t use gitpython.

But after trying multiple methods, I finally got this working. Note: It uses a few args from Jenkins && the point of this script is nothing more than to modify a file and commit it. With this functionality, I can now do real work.

import git
import os
import time
import yaml
import sys
import re
from git import Repo
print os.environ["GIT_BRANCH"]
workspace=os.environ['WORKSPACE']
m = re.search("origin/(.*)", os.environ["GIT_BRANCH"])
if m:
    git_branch = m.group(1)
else:
    sys.exit("COULD NOT LOAD SOURCE BRANCH")
# UPDATE VERSION FILE
with open(workspace + '/deploy_automation/config/int/versions.yaml', 'r') as f:
    versions_yaml = yaml.load(f)
versions_yaml["a.component"] = time.time()
with open(workspace + '/deploy_automation/config/int/versions.yaml', 'w') as f:
    yaml.dump(versions_yaml, f, default_flow_style=False)
# OTHER WAY
git_repo = Repo(workspace + "/deploy_automation")
git_repo.git.status()
git_repo.git.add(workspace + '/deploy_automation/config/int/versions.yaml')
git_repo.git.config('--global', "user.name", "user name")
git_repo.git.config('--global', "user.email", "user@domain.com")
git_repo.git.status()
git_repo.git.commit(m=' DEPLOY SCRIPT Updating versions.yaml for ENV jamestest2 and Service test')
git_repo.git.push('--set-upstream', 'origin', git_branch)

					

Python debugger

If you want to use Python’s debugger, put in your code:

import ipdb;ipdb.set_trace()

Now it will goto debug mode in said file or object.

Install PyYAML on OSX – Mountain Lion

When trying to setup pyyaml on Mountain Lion, I kept seeing a yaml.h error:

my-macbook:reorganization me$ pip install pyyaml
Downloading/unpacking pyyaml
  Downloading PyYAML-3.10.tar.gz (241kB): 241kB downloaded
  Running setup.py egg_info for package pyyaml
    
Installing collected packages: pyyaml
  Running setup.py install for pyyaml
    checking if libyaml is compilable
    cc -fno-strict-aliasing -fno-common -dynamic -I/usr/local/include -I/usr/local/opt/sqlite/include -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/usr/local/Cellar/python/2.7.5/Frameworks/Python.framework/Versions/2.7/include/python2.7 -c build/temp.macosx-10.8-x86_64-2.7/check_libyaml.c -o build/temp.macosx-10.8-x86_64-2.7/check_libyaml.o
    build/temp.macosx-10.8-x86_64-2.7/check_libyaml.c:2:10: fatal error: 'yaml.h' file not found
    #include 
             ^
    1 error generated.
    
    libyaml is not found or a compiler error: forcing --without-libyaml
    (if libyaml is installed correctly, you may need to
     specify the option --include-dirs or uncomment and
     modify the parameter include_dirs in setup.cfg)
    
Successfully installed pyyaml
Cleaning up...

Finally, I saw and did:

my-macbook:reorganization me$ sudo easy_install pip
my-macbook:reorganization me$ sudo brew install libyaml
my-macbook:reorganization me$ sudo easy_install setuptools
my-macbook:reorganization me$ pip install -U PyYAML

Migrating from a mysql DB to WordPress

My wife had a website of a recipe DB. I wanted to migrate it to WordPress automatically.

#!/usr/bin/python
# -*- coding: utf-8 -*-
import MySQLdb
import datetime, xmlrpclib
import sys
import string
wp_url = "http://www.website.com/wordpress/xmlrpc.php"
wp_username = "user"
wp_password = "password"
wp_blogid = ""

html_escape_table = {
"&": "&",
'"': """,
"'": "'",
">": ">",
"<": "&lt;",
"’": "&apos;",
}

def html_escape(text):
"""Produce entities within text."""
return "".join(html_escape_table.get(c,c) for c in text)

db = MySQLdb.connect(host="localhost", # your host, usually localhost
user="user", # your username
passwd="password", # your password
db="oohaheats") # name of the data base

# you must create a Cursor object. It will let
# you execute all the query you need
cur = db.cursor()
server = xmlrpclib.ServerProxy(wp_url)
cur.execute("SELECT id, name, created_at from recipes where active = 1");

recipe_rows = cur.fetchall()
for row in recipe_rows :
recipe_id = row[0]
recipe_name = row[1]
date_start = row[2]
categories = []

title = html_escape(recipe_name)
date_created = xmlrpclib.DateTime(date_start)

status_draft = 1
status_published = 0

# Load categories
cur.execute("select name from categories where id in (select category_id from categorizations where recipe_id = " + str(recipe_id) + ")")
category_rows = cur.fetchall()
for c_row in category_rows:
categories.append(c_row[0])
#print categories.size()
tags = categories

# Take care of ingredients.
cur.execute("SELECT measurement_id, grocery_id from ingredients where recipe_id = " + str(recipe_id) + " order by order_num")
ingredient_rows = cur.fetchall()

content = "<strong>Ingredients</strong>\n"
content += "<ul>\n"

for i_row in ingredient_rows:
measurement_id = i_row[0]
grocery_id = i_row[1]
grocery_name = ""
measurement_name = ""
content += "<li>"
cur.execute("SELECT name from measurements where id = " + str(measurement_id))
measurement_rows = cur.fetchall()
for m_row in measurement_rows:
measurement_name = m_row[0]
content += html_escape(measurement_name) + " "
cur.execute("SELECT name from groceries where id = " + str(grocery_id))
grocery_rows = cur.fetchall()
for g_row in grocery_rows:
grocery_name = html_escape(g_row[0])
content += html_escape(grocery_name)
content += "</li>\n"
content += "</ul>\n"

content += "<div><strong>Steps</strong></div>\n"
content += "<ol>\n"

cur.execute("select instruction from steps where recipe_id = " + str(recipe_id) + " order by order_num")
steps_rows = cur.fetchall()
for s_row in steps_rows:
instruction = s_row[0]
#print instruction
if len(instruction) > 0 and not all(c in string.whitespace for c in instruction):
content += "<li>" + html_escape(instruction) + "</li>\n"
content += "</ol>\n"
content += "</div>\n"

data = {'title': title, 'description': content, 'dateCreated': date_created, 'categories': categories, 'mt_keywords': tags}
try:
post_id = server.metaWeblog.newPost(wp_blogid, wp_username, wp_password, data, status_published)
except Exception, e:
print("ERROR POSTING " + str(recipe_id) + " = " + recipe_name)
print str(e)
print "-----------------------------"
sys.exit(1)

Python script – Jenkins job copy

This is a script which uses the XML parsing tool ElementTree to copy one Jenkins job to another & replaces the SVN location to be the latest created tag.

import urllib
import urllib2
from urllib2 import URLError
import logging
import re
import os
import base64
import sys
import time
import pysvn
import xml.etree.ElementTree
from xml.etree.ElementTree import ElementTree, Element, SubElement

# ElementTree XML tutorial -> http://www.bigfatalien.com/?p=223

"""
get_svn_login is required for pysvn to set credentials
"""
def get_svn_login(realm, username, may_save):
   return True, "svninfoforbuild", "svninfoforbuild", False
   
"""
query SVN for a list of tags. Look through that list for tags matching regex. 
figure out which of these was created last. return this tag.
"""
def svn_get_latest_tag(device, client):
   logging.debug( "svn_get_latest_tag(" + device + ", Client)")
   reponame = svn_url + "/" + svn_repo_list[device] + "/tags"
   logging.debug( "reponame = " + reponame)
   taglist = client.list(reponame)
   logging.debug("taglist = " + str(taglist))
   maxtime = 0
   maxtag = None
   tagname = None
   for tag in taglist:
      #for x in tag[0]:
      #   logging.debug( str(x) + " = " + str(tag[0][x]))
      m = re.search(r"/(" + device + "-d+.d+.d+.d+)$", tag[0]["path"])
      if m is not None:
         if tag[0]["time"] > maxtime:
            tagname = m.group(1)
            maxtime = tag[0]["time"]
            maxtag = tag[0]["path"]
   logging.debug( "maxtime = " + str(maxtime))
   logging.debug( "maxtag = " + str(maxtag))
   logging.debug( "tagname = " + str(tagname))
   return maxtime, maxtag, tagname

"""
query SVN for a list of branchs. Look through that list for branchs matching regex. 
figure out which of these was created last. return this branch.
"""
def svn_get_latest_branch(device, client):
   logging.debug( "svn_get_latest_tag(" + device + ", Client)")
   reponame = svn_url + "/" + svn_repo_list[device] + "/branches"
   logging.debug( "reponame = " + reponame)
   branchlist = client.list(reponame)
   maxtime = 0
   maxtag = None
   branchname = None
   for branch in branchlist:
      logging.debug( "------------------------")
      for x in branch[0]:
         logging.debug( str(x) + " = " + str(branch[0][x]))
      logging.debug( str(branch[0]["time"]) + " > " + str(maxtime))
      m = re.search(r"/(" + device + "-d+.d+.d+.[dw]+)$", branch[0]["path"])
      if m is not None:
         logging.debug( "m is not None")
         if branch[0]["time"] > maxtime:
            branchname = m.group(1)
            maxtime = branch[0]["time"]
            maxbranch = branch[0]["path"]
            logging.debug( "********* branchname == " + branchname)
            logging.debug( "------ maxtime == " + str(maxtime))
      logging.debug( "------------------------")
      
   logging.debug( "maxtime = " + str(maxtime))
   logging.debug( "maxbranch = " + str(maxbranch))
   logging.debug( "branchname = " + str(branchname))
   return maxtime, maxbranch, branchname

def jenkins_get_project_xml(device, timestr):
   jobxmlurl = jenkins_url + "/job/" + jenkins_project_list[device] + "/config.xml"
   req = urllib2.Request(jobxmlurl)
   base64string = base64.encodestring("%s:%s" % (jenkins_id, jenkins_pwd))[:-1]
   req.add_header("Authorization", "Basic %s" % base64string)
   response = urllib2.urlopen(req)
   jobxml = response.read()
   f = open("./" + device + "_config_" + timestr + ".xml", "w")
   f.write(jobxml)
   f.close()
   return "./" + device + "_config_" + timestr + ".xml"
   #logging.debug( jobxmlurl)

def jenkins_push_project_xml(new_proj_name, source_proj, new_proj_xml):
   # READ XML INTO MEMORY
   xml_contents = ""
   file = open(new_proj_xml, "r")
   for line in file:
      xml_contents += line
   file.close()
   
   # 1) CREATE A NEW JOB IN JENKINS
   job_create_url = jenkins_url + "/createItem?name=" + new_proj_name
   logging.debug( job_create_url)
   req = urllib2.Request(job_create_url, data=xml_contents, headers={"Content-Type":"text/xml"})
   base64string = base64.encodestring("%s:%s" % (jenkins_id, jenkins_pwd))[:-1]
   req.add_header("Authorization", "Basic %s" % base64string)
   response = urllib2.urlopen(req)
   create_results = response.read()
   
def jenkins_project_exists(pname):
   logging.debug("jenkins_project_exists(" + pname + ")")
   jobxmlurl = jenkins_url + "/job/" + pname
   logging.debug( "jobxmlurl = " + jobxmlurl)
   req = urllib2.Request(jobxmlurl)
   base64string = base64.encodestring("%s:%s" % (jenkins_id, jenkins_pwd))[:-1]
   req.add_header("Authorization", "Basic %s" % base64string)
   try:
      response = urllib2.urlopen(req)
   except URLError, e:
      return False
   return True
   
def svn_get_trunk_url(device):
   return svn_url + "/" + svn_repo_list[device] + "/trunk"

def process_xml(device, tagname, jobxml, svn_trunk_url, svn_maxtag_url, timestr):
   
   # Element - XML elements containing:
   #           the label (tag), 
   #           a list of attributes
   #           a list of chilren elements (forming the XML tree hierarchy)
   # Elementtree - The wrappers around Element objects which provide facilities to output the Element as an xml file
   #               You can also read an xml file into an ElementTree then access teh Elements within.
   
   # CREATE AN ELEMENT TREE OBJECT
   et = ElementTree()
   # PARSE THE XML FILE.
   project_elem = et.parse(jobxml)
   
   
   logging.debug( "--------------------------------------------")
   # REPLACE THE SCM LOCATION POINTING TO TRUNK WITH ONE POINTING TO THE NEW TAG
   # Structure = 
   #
   #  ...
   #  projectname
   #  ...
   #  
   #     
   #        
   #           ...
   #           ...
   #        
   #        ...
   #     
   #     ...
   #  
   #  ...
   #
   # FIND THE FIRST description ELEMENT 
   de = project_elem.find("description")
   de.text = device + " TAG " + tagname
   scme = project_elem.find("scm")
   loce = scme.find("locations")
   location_children = list(loce)
   logging.debug( "svn_trunk_url = " + svn_trunk_url)
   for l in location_children:
      remote = l.find("remote")
      local = l.find("local")
      m = re.search(r"^" + svn_trunk_url + "(/src)?$", remote.text) 
      if m is not None:
         newstr = svn_maxtag_url
         if m.group(1) is not None:
            newstr += m.group(1)
         logging.debug( newstr)
         remote.text=newstr
   newxmlname = "./" + device + "_config_new_" + timestr + ".xml" 
   et.write(newxmlname);
   return(newxmlname);
   
   
   
svn_url="https://svn.company.com"
#device_type="ios"
jenkins_url="http://jenkins.corp.company.com:8080"
job_sub_http="job"

svn_repo_list={}
svn_repo_list["android"] = "android"
svn_repo_list["ios"] = "ios"
svn_repo_list["blackberry"] = "blackberry"

jenkins_project_list={}
jenkins_project_list["android"] = "android_trunk"
jenkins_project_list["ios"] = "ios_trunk"
jenkins_project_list["blackberry"] = "blackberry_trunk"

jenkins_id = "tag_creator"
jenkins_pwd = "1qazxsw2"

job_file="config.xml"

logging.basicConfig(level=logging.DEBUG)

timestr = time.strftime("%Y-%m-%d_%H%M%S", time.localtime())

device_type = None
svn_loc = "tags"
for arg in sys.argv:
   args = re.search(r"^--(S+)=(S+)$", arg)
   if args is not None:
      if re.search(r"^device_type$", args.group(1)):
         device_type = args.group(2).lower()
         logging.debug("device_type = " + device_type)
      elif re.search(r"^svn_loc$", args.group(1)):
         svn_loc = args.group(2)
         logging.debug("svn_loc = " + svn_loc)


try:
   repo = svn_repo_list[device_type]
   logging.debug("repo = " + repo)
except KeyError, e:
   sys.exit("ERROR: device_type does not match the required list of options (android, ios, blackberry)")
   
svn_trunk_url=svn_url +  repo + "/trunk"
logging.debug("svn_trunk_url = " + svn_trunk_url)

client = pysvn.Client()
client.callback_get_login = get_svn_login

maxtime = None
svn_maxtag_url = None
reponame = None


if svn_loc == "tags":
   maxtime, svn_maxtag_url, reponame =  svn_get_latest_tag(device_type, client)
elif svn_loc == "branches":
   maxtime, svn_maxtag_url, reponame =  svn_get_latest_branch(device_type, client)
else:
   sys.exit("ERROR: You cannot use svn_loc = " + svn_loc)
   
logging.debug("maxtime = " + str(maxtime))
logging.debug("svn_maxtag_url = " + str(svn_maxtag_url))
logging.debug("reponame = " + str(reponame))


project_exists = jenkins_project_exists(reponame)
logging.debug("project_exists = " + str(project_exists))
if project_exists:
   sys.exit("ERROR: The Jenkins job " + reponame + " already exists.")
   
origxml = jenkins_get_project_xml(device_type, timestr)
newxml = process_xml(device_type, reponame, origxml, svn_trunk_url, svn_maxtag_url, timestr)
jenkins_push_project_xml(reponame, jenkins_project_list[device_type], newxml)

os.remove(origxml)
os.remove(newxml)