#!/usr/bin/env python3 import argparse import datetime import importlib.util import os import os.path import re import shutil import sqlite3 import subprocess import sys import textwrap known_modules = ["jpeg", "szip", "gsl"] if os.environ.get('OCSSW_MPI') == '1': known_modules.append('openmpi') known_modules.extend(["hdf4", "hdf5", "netcdf", "sqlite", "libpng", "tiff", "proj4", "geotiff", "fann", "grib", "cmpfit", "lapack", "levmar", "pugixml", "boost", "openjpeg", "jansson", "rapidjson", "ms-gsl", "fftw", "geographiclib", "gmp", "mpfr", "cgal"]) known_modules.extend(["hdfeos", "hdfeos5", "sdptk", "mapi", "sdst"]) if os.environ.get('OCSSW_GDAL') == '1': known_modules.append('gdal') known_zip_extensions = tuple(['.tar.Z', '.tgz', '.tar.gz', '.tbz', '.tar.bz2', '.zip', '.tar']) class BuildIt(): def __init__(self): self.opt_src = os.path.abspath(os.path.dirname(__file__) or ".") self.opt_dir = os.path.dirname(self.opt_src) db_path = "%s/.buildit.db" % self.opt_src self.dbh = sqlite3.connect(db_path, isolation_level=None) if not os.stat(db_path).st_size: self.dbh.execute("CREATE TABLE installed (module TEXT, version TEXT, date TEXT)") self.dbh.execute("CREATE INDEX installed_idx_module ON installed (module)") pass def setup_environment(self): arch = os.environ['OCSSW_ARCH'] cflags = ['-I%s' % (os.path.join(self.opt_dir, 'include'))] cxxflags = ['-I%s' % (os.path.join(self.opt_dir, 'include'))] fflags = [] ldflags = ['-Wl,-rpath=%s/lib,-rpath=\$$ORIGIN/../lib,-rpath=\$$ORIGIN/../opt/lib' % (self.opt_dir), '-L%s' % (os.path.join(self.opt_dir, 'lib'))] f77 = os.environ['FC'] f9x = f77 gcc_version = subprocess.run(['gcc', '-dumpversion'], stdout=subprocess.PIPE) if gcc_version.stdout.decode('utf-8') >= '6': cflags.append('-Wno-error=misleading-indentation') if arch == "linux_64": os.environ['LINUX_BRAND'] = 'linux64' if os.environ['FC'] == 'ifort': f9x = 'ifort -fpp -DDEC$=DEC_ -DMS$=MS_' fflags.append("-m64 -extend_source -convert big_endian -assume byterecl -save") else: fflags.append("-m64 -ffixed-line-length-none -fconvert=big-endian -fno-automatic %s" % (os.environ['GCC_TUNE'])) elif arch == "linux": os.environ['LINUX_BRAND'] = 'linux' fflags.append("-ffixed-line-length-none -fconvert=big-endian -fno-automatic %s" % (os.environ['GCC_TUNE'])) elif arch == "macosx_intel": ldflags = ["-L%s/lib" % (self.opt_dir), "-Wl,-rpath,%s/lib,-rpath,@executable_path/../lib,-rpath,@executable_path/../opt/lib" % (self.opt_dir)] fflags.append("-ffixed-line-length-none -fconvert=big-endian -fno-automatic %s" % (os.environ['GCC_TUNE'])) elif arch == "macosx_ppc": fflags.append("-ffixed-line-length-none -fno-automatic") else: os.environ['LINUX_BRAND'] = 'linux' fflags.append("-ffixed-line-length-none -fconvert=big-endian -fno-automatic") if os.environ['OCSSW_DEBUG'] == "1": cflags.append("-g3 -gdwarf-2") fflags.append("-g3 -gdwarf-2") cxxflags.append("-g3 -gdwarf-2") else: cflags.append("-O3") fflags.append("-O3") cxxflags.append("-O3") os.environ['CFLAGS'] = ' '.join(cflags) os.environ['CXXFLAGS'] = ' '.join(cxxflags) os.environ['FFLAGS'] = ' '.join(fflags) os.environ['LDFLAGS'] = ' '.join(ldflags) os.environ['F77'] = f77 os.environ['F9X'] = f9x def run(self): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity") parser.set_defaults(func=self.__build) subparsers = parser.add_subparsers() self.__add_subparser_build(subparsers) self.__add_subparser_list(subparsers) self.__add_subparser_generate(subparsers) self.__add_subparser_clean(subparsers) options, args = parser.parse_known_args() options.func(options, args) if os.environ['OCSSW_ARCH'] == "macosx_intel": if(options.func == self.__build): self.fix_mac_rpath(options) def __add_subparser_build(self, subparsers): parser_build = subparsers.add_parser('build') parser_build.add_argument("-a", "--all", default=False, action="store_true", help="build all 3rd party libs, ignoring what is already installed in the system") def __add_subparser_list(self, subparsers): parser_list = subparsers.add_parser('list') parser_list.set_defaults(func=self.__list) def __add_subparser_generate(self, subparsers): parser_gen = subparsers.add_parser('generate') type_group = parser_gen.add_mutually_exclusive_group() type_group.add_argument("-c", "--cmake", action="store_const", dest="type_func", const=self.__generate_cmake, help="generate build script for CMake") type_group.add_argument("-m", "--makefile", action="store_const", dest="type_func", const=self.__generate_makefile, help="generate build script for Makefile") parser_gen.add_argument("--prefix", default=self.opt_dir, help="set install prefix") parser_gen.add_argument("module_name", help="new module name") parser_gen.set_defaults(func=self.__generate) def __add_subparser_clean(self, subparsers): parser_clean = subparsers.add_parser('clean') parser_clean.set_defaults(func=self.__clean) def __get_installed_version(self, mod_name): cursor = self.dbh.cursor() cursor.execute("SELECT version FROM installed WHERE module = ?", [mod_name]) row = cursor.fetchone() if row: return row[0] return def __list(self, options, modules): if not modules: modules = known_modules for mod_name in modules: mod = self.__load_module(mod_name) if mod: installed_version = self.__get_installed_version(mod_name) mod_version = mod.version(self) print("%s %s (%s installed)" % (mod_name, mod_version, installed_version or 'not')) else: print("Failed to load module %s" % mod_name) def __clean(self, options, modules): if modules: print("clean takes no arguments") else: for d in (map(lambda d: os.path.join(os.environ["LIB3_DIR"], d), ("bin", "cmake", "EOS", "examples", "include", "lib", "share"))): if os.path.isdir(d): print("Deleting %s" % d) shutil.rmtree(d) else: print("%s not found, skipping" % d) if os.path.isfile(os.path.join(os.environ["LIB3_DIR"], "src", ".buildit.db")): os.remove(os.path.join(os.environ["LIB3_DIR"], "src", ".buildit.db")) def restrict_modules(self, modules): build_dir = "%s/build" % self.opt_src if os.path.isdir(build_dir): shutil.rmtree(build_dir) os.mkdir(build_dir) out_str = subprocess.check_output("cmake ..", shell=True, cwd=build_dir) shutil.rmtree(build_dir) out_str = out_str.decode("utf-8") for line in out_str.splitlines(): if "LIB3 Package Installed:" in line: line = line.split(":")[1].strip() if "TRUE" in line: word = line.split("=")[0] if word in modules: modules.remove(word) return modules def fix_mac_rpath(self, options): # set the rpath in the libs os.chdir(os.path.join(self.opt_dir, "lib")) for fileName in os.listdir('.'): if os.path.isfile(fileName): if ".dylib" in fileName: #print (fileName) p = subprocess.Popen(["otool", "-D", fileName], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() out = out.decode("utf-8") err = err.decode("utf-8") #print("out=",out) parts = out.split() #print(parts) parts = parts[1].split("/") if not "@rpath" in parts[0]: name = parts[len(parts)-1] id = "@rpath/" + name # print(fileName, id) subprocess.call(["install_name_tool", "-id", id, fileName]) p = subprocess.Popen(["otool", "-L", fileName], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() out = out.decode("utf-8") err = err.decode("utf-8") #print("out=",out) lines = out.split('\n') for line in lines: if os.environ['LIB3_DIR'] in line: #print(' ' + line) libPath = line.split()[0] parts = libPath.split('/') libName = parts[len(parts)-1] newName = '@rpath/' + libName #print(' ' + libPath + ' -> ' + newName) subprocess.call(["install_name_tool", "-change", libPath, newName, fileName]) # set the rpath in the executables os.chdir(os.path.join(self.opt_dir, "bin")) for fileName in os.listdir('.'): if os.path.isfile(fileName): line = subprocess.check_output(['file', fileName]) if b"Mach-O 64-bit executable" in line: #print ('------' + fileName) p = subprocess.Popen(["otool", "-L", fileName], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() out = out.decode("utf-8") err = err.decode("utf-8") #print("out=",out) lines = out.split('\n') for line in lines: if os.environ['LIB3_DIR'] in line: #print(' ' + line) libPath = line.split()[0] parts = libPath.split('/') libName = parts[len(parts)-1] newName = '@rpath/' + libName #print(' ' + libPath + ' -> ' + newName) subprocess.call(["install_name_tool", "-change", libPath, newName, fileName]) def __build(self, options, modules): if not modules: modules = known_modules #if (not hasattr(options, "all")) or (not options.all): #modules = self.restrict_modules(modules) self.setup_environment() for mod_name in modules: print("Working on %s" % mod_name) mod = self.__load_module(mod_name) if mod: mod_version = mod.version(self) installed_version = self.__get_installed_version(mod_name) if installed_version and installed_version == mod_version: print("%s %s already installed" % (mod_name, mod_version)) print("Delete %s/.buildit.db to overwrite all installs" % self.opt_src) else: os.chdir(os.path.join(self.opt_src, mod_name)) try: if installed_version: #mod.uninstall(self) print("%s version already installed, hopefully overwriting with new install" % installed_version) mod.install(self) if installed_version: self.dbh.execute("UPDATE installed SET version=?, date=? WHERE module=?", [mod_version, str(datetime.datetime.now()), mod_name]) else: self.dbh.execute("INSERT INTO installed VALUES (?, ?, ?)", [mod_name, mod_version, str(datetime.datetime.now())]) except: print("Unknown error encountered installing %s" % mod_name) return 1 else: print("Failed to load module %s, guessing install" % mod_name) mod_dir = os.path.join(self.opt_src, mod_name) os.chdir(mod_dir) zips = [f for f in os.listdir(mod_dir) if f.endswith(known_zip_extensions)] for zip_ in zips: self.extract_file(zip_) zip_no_ext = self.remove_zip_extension(zip_) os.chdir(zip_no_ext) for patch_dir in (["patches-" + zip_no_ext, "patches"]): patch_dir_full = os.path.join(mod_dir, patch_dir) if os.path.isdir(patch_dir_full): for patch in os.listdir(patch_dir_full): with open(os.path.join(patch_dir, patch), 'r') as patch_h: subprocess.check_call(["patch", "-b", "-p0"], stdin=patch_h) if os.path.isfile('CMakeLists.txt'): if not os.path.isdir("build"): os.makedirs("build") os.chdir("build") subprocess.check_call(["cmake", "..", "-DCMAKE_PREFIX_PATH=%s" % (self.opt_dir), "-DCMAKE_INSTALL_PREFIX=%s" % (self.opt_dir), "-DBUILD_SHARED_LIBS=ON"]) subprocess.check_call(["cmake", "--build", ".", "--config", "Release"]) subprocess.check_call(["cmake", "--build", ".", "--target", "install"]) elif os.path.isfile('configure'): subprocess.check_call(["./configure", "--prefix", self.opt_dir]) subprocess.check_call(["make"]) subprocess.check_call(["make", "install"]) else: raise(Exception("No good guesses could be made for building")) os.chdir(mod_dir) if not os.environ['OCSSW_DEBUG'] or os.environ['OCSSW_DEBUG'] == "0": shutil.rmtree(zip_no_ext) #if os.environ['OCSSW_ARCH'] == "macosx_intel": # self.fix_mac_rpath(options) os.chdir(self.opt_src) print("\nEverything Built Successfully\n") return 0 def __guess_version(self, zips): version_guess = ' '.join(re.sub("^.*?-([^-]+)$", r"\1", self.remove_zip_extension(zip_)) for zip_ in zips) if not len(version_guess): version_guess = "unknown" return version_guess def __generate(self, options, args): #if args: #print("Erroneous arguments given to generate, ignoring (%s)" % args) print("Generating build script for %s" % options.module_name) mod_dir = os.path.join(self.opt_src, options.module_name) mod_path = os.path.join(mod_dir, "BuildIt.py") if os.path.isfile(mod_path): print("%s already exists" % mod_path) #return 1 if not os.path.isdir(mod_dir): os.makedirs(mod_dir) zips = [f for f in os.listdir(mod_dir) if f.endswith(known_zip_extensions)] if not zips: zips = ["example-1.1.1.tgz"] with open(mod_path, 'w') as dest_h: dest_h.write(textwrap.dedent(''' #!/usr/bin/env python3 import importlib.util import os import os.path import shutil import subprocess import sys this_dir = os.path.abspath(os.path.dirname(__file__) or ".") def version(build_it): ''').lstrip()) version_guess = self.__guess_version(zips) dest_h.write(''' return "%s"\n''' % (version_guess)) dest_h.write(textwrap.dedent(''' def install(build_it): ''')) for zip_ in zips: zip_no_ext = self.remove_zip_extension(zip_) dest_h.write(re.sub("^ {16}", "", '''\ zip_ = "%s" src_dir = "%s" if os.path.isdir(src_dir): shutil.rmtree(src_dir) build_it.extract_file(zip_) os.chdir(src_dir) ''' % (zip_, zip_no_ext), flags=re.M)) for patch_dir in (["patches-" + zip_no_ext, "patches"]): patch_dir_full = os.path.join(mod_dir, patch_dir) if os.path.isdir(patch_dir_full): dest_h.write(''' patch_dir = os.path.join(this_dir, "%s")\n''' % (patch_dir)) for patch in os.listdir(patch_dir_full): dest_h.write(re.sub("^\\s{28}", "", '''\ with open(os.path.join(patch_dir, "%s"), 'r') as patch_h: subprocess.check_call(["patch", "-b", "-p0"], stdin=patch_h) ''' % (patch), flags=re.M)) dest_h.write("\n") if options.type_func: options.type_func(options, dest_h, zip, args) else: self.__generate_makefile(options, dest_h, zip, args) dest_h.write("\n") self.__generate_cmake(options, dest_h, zip, args) dest_h.write("\n") dest_h.write(re.sub("^ {16}", "", '''\ os.chdir(this_dir) if not os.environ['OCSSW_DEBUG'] or os.environ['OCSSW_DEBUG'] == "0": shutil.rmtree(src_dir) ''', flags=re.M)) dest_h.write(textwrap.dedent(''' if __name__ == "__main__": os.chdir(this_dir) spec = importlib.util.spec_from_file_location("BuildIt", os.path.join("..", "BuildIt.py")) build_it = importlib.util.module_from_spec(spec) spec.loader.exec_module(build_it) master = build_it.BuildIt() master.setup_environment() sys.exit(install(master)) ''')) return 0 def remove_zip_extension(self, zip_): for ext in known_zip_extensions: if zip_.endswith(ext): return re.sub("%s$" % ext, "", zip_) return zip_ def extract_file(self, zip_): print("Extracting %s" % zip_) try: if zip_.endswith(".zip"): subprocess.check_call(["unzip", zip_]) elif zip_.endswith(".tar"): subprocess.check_call(["tar", "-xf", zip_]) elif zip_.endswith(".tar.gz") or zip_.endswith(".tgz") or zip_.endswith(".tar.Z"): subprocess.check_call(["tar", "-xzf", zip_]) elif zip_.endswith(".tar.bz2") or zip_.endswith(".tbz"): subprocess.check_call(["tar", "-xjf", zip_]) else: raise(Exception("Invalid zip format")) except: print("Error while extracting %s (%s)" % (zip_, sys.exc_info()[0])) return 1 return def __load_module(self, mod_name): mod_path = os.path.join(self.opt_src, mod_name, "BuildIt.py") if os.path.isfile(mod_path): spec = importlib.util.spec_from_file_location(mod_name, mod_path) mod_obj = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod_obj) return mod_obj else: print("%s has no BuildIt.py" % mod_name) return def __generate_makefile(self, options, dest_h, zip_, args): dest_h.write(re.sub("^ {8}", "", '''\ subprocess.check_call(["./configure", "--prefix", build_it.opt_dir]) subprocess.check_call(["make"]) #subprocess.check_call(["make", "test"]) subprocess.check_call(["make", "install"]) ''', flags=re.M)) def __generate_cmake(self, options, dest_h, zip_, args): dest_h.write(re.sub("^ {8}", "", '''\ os.makedirs("build") os.chdir("build") subprocess.check_call(["cmake", "..", "-DCMAKE_PREFIX_PATH=%s" % (build_it.opt_dir), "-DCMAKE_INSTALL_PREFIX=%s" % (build_it.opt_dir), "-DBUILD_SHARED_LIBS=ON"]) subprocess.check_call(["cmake", "--build", ".", "--config", "Release"]) subprocess.check_call(["cmake", "--build", ".", "--target", "install"]) ''', flags=re.M)) if __name__ == "__main__": sys.exit(BuildIt().run())