Multithreading added in the compile script

util/compile.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os, sys, shutil, platform, getpass, subprocess, ftplib, time
+import os, sys, shutil, platform, getpass, subprocess, ftplib, time, threading
 
 BINARY_DIR = "../binaries/"
 
@@ -33,6 +33,7 @@ user = "open9x"
 password = None
 ftp_connection = None
 ftp_tmpdir = None
+ftp_lock = threading.Lock()
 
 def openFtp():
     global password
@@ -53,6 +54,7 @@ def closeFtp():
     ftp_connection.quit()
     
 def uploadBinary(binary_name):
+    ftp_lock.acquire()
     while 1:
         try:
             try:
@@ -62,6 +64,7 @@ def uploadBinary(binary_name):
             f = file(BINARY_DIR + binary_name, 'rb') 
             ftp_connection.storbinary('STOR ' + ftp_tmpdir + '/' + binary_name, f)
             f.close()
+            ftp_lock.release()
             return
         except:
             time.sleep(10)
@@ -74,21 +77,24 @@ def uploadBinary(binary_name):
             except:
                 pass   
 
+global_current = 0
+global_count = 0 
+def generate(hex, arg, extension, options, languages, maxsize):
+    global global_current
+    global global_count
     
-def generate(hex, arg, extension, options, maxsize):
     result = []
     states = [0] * len(options)
     
     count = len(languages)
     for option in options:
       count *= len(option)
-    current = 0
+    global_count += count
     
     while 1:
         # print index, states
     
         for language in languages:
-            current += 1
             hex_file = hex
             make_args = ["make", arg]
             for i, option in enumerate(options):
@@ -98,11 +104,16 @@ def generate(hex, arg, extension, options, maxsize):
                 make_args.append(option[state][1])
             hex_file += "-" + language
             make_args.append("TRANSLATIONS=" + language.upper())
-            print "[%d/%d]" % (current, count), hex_file
-            subprocess.check_output(["make", "clean", arg])
-            p = subprocess.Popen(make_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if mt:
+                cwd = language
+            else:
+                cwd = None
+            subprocess.Popen(["make", "clean", arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd).wait()
+            p = subprocess.Popen(make_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
             p.wait()
             stderr = p.stderr.read()
+            global_current += 1
+            print "[%d/%d]" % (global_current, global_count), hex_file
             if "error" in stderr:
                 print stderr
                 exit()
@@ -125,6 +136,9 @@ def generate(hex, arg, extension, options, maxsize):
             
             if size <= maxsize:
                 binary_name =  hex_file + "." + extension
+                if mt:
+                    shutil.copyfile(language + "/open9x." + extension, BINARY_DIR + binary_name)
+                else:
                     shutil.copyfile("open9x." + extension, BINARY_DIR + binary_name)
                 if upload:
                     uploadBinary(binary_name)
@@ -142,6 +156,35 @@ def generate(hex, arg, extension, options, maxsize):
         
     return result
 
+class GenerateThread(threading.Thread):
+   def __init__ (self, hex, arg, extension, options, language, maxsize):
+      threading.Thread.__init__(self)
+      self.hex = hex
+      self.arg = arg
+      self.extension = extension
+      self.options = options
+      self.language = language
+      self.maxsize = maxsize
+      
+   def run(self):
+      self.hexes = generate(self.hex, self.arg, self.extension, self.options, [self.language], self.maxsize)     
+           
+def multithread_generate(hex, arg, extension, options, languages, maxsize):
+    if mt:
+        result = []
+        threads = []
+        for language in languages:
+            thread = GenerateThread(hex, arg, extension, options, language, maxsize)
+            threads.append(thread)
+            thread.start()
+        for thread in threads:
+            thread.join()
+            result.extend(thread.hexes)
+        return result
+    else:
+        return generate(hex, arg, extension, options, languages, maxsize)
+        
+        
 def generate_c9x_list(filename, hexes, extension, stamp, board):
     f = file(filename, "w")
     for hex in hexes:
@@ -150,13 +193,20 @@ def generate_c9x_list(filename, hexes, extension, stamp, board):
 if __name__ == "__main__":
     
     upload = "upload" in sys.argv
+    mt = "mt" in sys.argv and platform.system() != "Windows"
     
     if upload:
         openFtp()
 
+    if mt:
+        for lang in languages:
+            print "Directory %s creation..." % lang
+            shutil.rmtree(lang, True)
+            shutil.copytree("../src", lang)
+    
     if platform.system() == "Windows":
         # arm board
-        hexes = generate("open9x-arm", "PCB=ARM", "bin", options_arm, 262000)
+        hexes = generate("open9x-arm", "PCB=ARM", "bin", options_arm, languages, 262000)
         generate_c9x_list("../../companion9x/src/open9x-arm-binaries.cpp", hexes, "bin", "OPEN9X_ARM_STAMP", "BOARD_ERSKY9X")
         
         # arm stamp
@@ -164,11 +214,11 @@ if __name__ == "__main__":
     
     else:
         # stock board
-        hexes = generate("open9x-stock", "PCB=STD", "hex", options_stock, 65530)
+        hexes = multithread_generate("open9x-stock", "PCB=STD", "hex", options_stock, languages, 65530)
         generate_c9x_list("../../companion9x/src/open9x-stock-binaries.cpp", hexes, "hex", "OPEN9X_STAMP", "BOARD_STOCK")
     
         # v4 board
-        hexes = generate("open9x-v4", "PCB=V4", "hex", options_v4, 262000)
+        hexes = multithread_generate("open9x-v4", "PCB=V4", "hex", options_v4, languages, 262000)
         generate_c9x_list("../../companion9x/src/open9x-v4-binaries.cpp", hexes, "hex", "OPEN9X_STAMP", "BOARD_GRUVIN9X")
     
         # stamp