from userconfig.tools import get_config from userconfig.checks import check_class import os from userconfig.tools import diff, user_config_generated, backup_file, copy_file import time import tempfile from pathlib import Path class Userconfig: _cfg = None def __init__(self, cfg): self._cfg = cfg """ This is NEW """ def process_package_dir(self, package_dir): """ Walk through package_dir, collect all directories inside, parse them according to our specs and return a sorted list with entries (prio, category, value, path) :param package_dir: root of package directory :return: list of tuples (prio, category, value, path), sorted with prio """ self._cfg.debug.stdout(f'*** process package {package_dir}', 3) package_config_file = f'{package_dir}/{self._cfg.get("configfile")}' if not os.path.isfile(package_config_file): self._cfg.debug.stdout(f'No config file {self._cfg.get("configfile")} in {package_dir}, skipping', 1, 'ERROR') return None dir_config = get_config(package_config_file, self._cfg) if not dir_config: self._cfg.debug.stdout(f'Cannot read config file {package_config_file}, skipping', 1, 'ERROR') return None classes = [] for category in os.scandir(package_dir): if category.path == package_config_file: continue if not category.is_dir(): self._cfg.debug.stdout(f'{category.path} is not a directory, skipping', 3, 'WARNING') continue self._cfg.debug.stdout(f'****** process category {category.path}', 3) content = category.name.split('_') # Format: __ if len(content) == 3: prio_string = content[0] category_name = content[1] value = content[2] elif len(content) == 2: prio_string = content[0] category_name = content[1] value = '' else: self._cfg.debug.stdout(f'Format of package directory {category.path} wrong, skipping', 1, 'ERROR') continue try: prio = int(prio_string) except ValueError: self._cfg.debug.stdout(f'Cannot convert prio to integer ({category.path}, skipping', 1, 'ERROR') continue # self._cfg.debug.stdout(f'****** Got class: {(prio, category_name, value)}', 3) classes.append((prio, category_name, value, category.path)) classes.sort(key=lambda k: k[0]) return classes, dir_config def filter_categories(self, categories): """ Get classes list of tuples from process_package_dir and filter it for host running the command :param categories: list of tuples from process_package_dir :return: matching classes for this host """ ret = [] for c in categories: if not c[1]: self._cfg.debug.stdout(f'category not set for {c[3]}, skipping', 3, 'WARNING') continue if check_class(c): ret.append(c) return ret def process_category_dir(self, category_dir_tuple, file_list): """ Walk through category_dir, collect all files :param category_dir_tuple: :param file_list: :return: """ (prio, category, value, category_dir) = category_dir_tuple self._cfg.debug.stdout(f'*** process category {category_dir}', 3) for file in os.scandir(category_dir): if file.name not in file_list: file_list[file.name] = [] file_list[file.name].append(file.path) return file_list def build_file(self, files, destfile, commentstring): """ merge all files in files[] and write contents into a tempfile :param files: :param destfile: :param commentstring: :return: tempfile name """ content = [] self._cfg.debug.stdout(f'*** building file for {destfile}', 3) if commentstring: content.append(f'{commentstring} {self._cfg.get("stamp")} {time.strftime("%+")}\n') for file in files: self._cfg.debug.stdout(f'Merging {file}', 3) fp = open(file, "r") filecontent = fp.read() fp.close() # if commentstring exists, add comment with category dir and filename if commentstring: content.append(f'{commentstring} {"/".join(file.split("/")[-2:])}') content.append(filecontent) (tempfd, tempfilename) = tempfile.mkstemp(prefix=os.path.basename(destfile), dir="/tmp") try: fp = os.fdopen(tempfd, "w") except Exception as e: self._cfg.debug.stderr(f"Cannot write to temporary file {tempfilename}: {e}", 0, 'ERROR') os.remove(tempfilename) return False self._cfg.debug.stdout(f'Writing merged files into tempfile {tempfilename}', 3) for block in content: fp.write(block) fp.write("\n") fp.close() return tempfilename def copy_file(self, temp_filename, dest_filename, commentstring): if diff(dest_filename, temp_filename, commentstring, self._cfg): self._cfg.debug.stdout(f"File {dest_filename} has changed", 0, 'NOTICE') if not user_config_generated(dest_filename, self._cfg): self._cfg.debug.stdout(f"{dest_filename} not generated by userconfig, backing up.", 3) backup_file(dest_filename, self._cfg) self._cfg.debug.stdout(f"Copy {temp_filename} to {dest_filename}", 0, 'NOTICE') copy_file(temp_filename, dest_filename, self._cfg) self._cfg.debug.stdout(f"Removing temporary file {temp_filename}", 3) os.remove(temp_filename) def create_destination_directories(self, dest_directory): self._cfg.debug.stdout(f'*** Creating {dest_directory} if needed', 3) path = Path(dest_directory) if not path.exists(): path.mkdir(parents=True) # def build_file_old(self, classfiles, destfile, commentstring): # """open all classfiles, assemble them and write the contents into a tempfile # returns the name of tempfile # :param classfiles: # :param destfile: # :param commentstring: # :return: str # """ # content = [] # self._cfg.debug.stdout(f" ================ build_file {destfile} ===============", 1) # if commentstring != "": # self._cfg.debug.stdout(" +++ commentstring found, adding header.", 3) # content.append(commentstring + " " + self._cfg.get("stamp") + " " + time.strftime("%+") + "\n") # # for f in classfiles: # self._cfg.debug.stdout(" +++ Merging %s." % f, 4) # fp = open(f, "r") # filecontent = fp.read() # fp.close() # if commentstring == "": # # look for stamp in content, replace with real stamp # if re.search(re.escape(self._cfg.get("stampreplace")), filecontent): # self._cfg.debug.stdout(" +++ commentstring empty, replacing stamp in file", 3) # filecontent = re.sub(re.escape(self._cfg.get("stampreplace")), self._cfg.get("stamp"), # filecontent) # content.append(filecontent) # # (tempfd, tempfilename) = tempfile.mkstemp(prefix=os.path.basename(destfile), dir="/tmp") # # try: # fp = os.fdopen(tempfd, "w") # except Exception as e: # self._cfg.debug.stderr(f"Cannot write to temporary file {tempfilename}: {e}") # os.remove(tempfilename) # return False # self._cfg.debug.stdout(" +++ Writing merged files into tempfile %s." % tempfilename, 3) # for block in content: # fp.write(block) # fp.write("\n") # fp.close() # self._cfg.debug.stdout(f" ================ done: build_file {destfile} ===============", 1) # return tempfilename # def process_all_files(self, destfiles, dir_config): # """processes all files in destfiles, generate files from classes, compare and copy if necessary""" # self._cfg.debug.stdout(" ================ process_all_files ===============", 1) # for df in destfiles.keys(): # self._cfg.debug.stdout(" ??? Processing source files for %s." % df, 2) # if not os.path.exists(os.path.dirname(df)): # self._cfg.debug.stdout(" +++ Directory %s does not exist, creating" % os.path.dirname(df), 1) # os.mkdir(os.path.dirname(df)) # if not os.path.isdir(os.path.dirname(df)): # self._cfg.debug.stdout(f" --- Destination directory {os.path.dirname(df)} does not exist, skipping.", # 1) # return False # # assemble file to tmp # commentstring = "" # if dir_config.check(section="Main", option="commentstring"): # commentstring = dir_config.get(section="Main", option="commentstring") # self._cfg.debug.stdout(" +++ Found commentstring %s in %s" % (commentstring, df), 3) # # tempfilename = self.build_file(destfiles[df], df, commentstring) # if not tempfilename: # self._cfg.debug.stdout(" --- Error while creating temp file for %s, skipping." % df, 1) # continue # self._cfg.debug.stdout(" +++ Merged files %s for %s into %s" % (str(destfiles[df]), df, tempfilename), 2) # # # diff assembled file and config file # if diff(df, tempfilename, commentstring, self._cfg): # self._cfg.debug.stdout("File %s has changed" % df, 0) # if not user_config_generated(df, self._cfg): # self._cfg.debug.stdout(" +++ %s not generated by userconfig, backing up." % df, 2) # # file not generated from userconfig -> back up # backup_file(df, self._cfg) # # copy tmp file to real location # self._cfg.debug.stdout("Copy %s to %s." % (tempfilename, df), 0) # copy_file(tempfilename, df, self._cfg) # # remove tmp # self._cfg.debug.stdout(" +++ Removing temporary file %s." % tempfilename, 2) # os.remove(tempfilename) # self._cfg.debug.stdout(" ================ process_all_files ===============", 1) # def workconf(self, directory, depth=2): # """walks through directory, collecting all filenames, returns list of all filenames""" # dirs = os.listdir(directory) # ret = [] # self._cfg.debug.stdout(f" ================ workconf {directory} ===============", 1) # for d in dirs: # name = directory + "/" + d # if os.path.isdir(name): # self.workconf(name, depth + 1) # if name.endswith(".swp"): # continue # if d == ".svn": # continue # ret.append(name) # self._cfg.debug.stdout(" +++ Found file %s in directory %s" % (name, directory), 4) # self._cfg.debug.stdout(f" ================ done: workconf {directory} ===============", 1) # return ret # def workdir(self, directory): # """walks through all host classes, checking if the classes directory exists in directory # then collect all filenames within this directory and return a dict of all files for this # directory # """ # self._cfg.debug.stdout(f" ================ workdir {directory} ===============", 1) # # skip directory if no CONFIGFILE present # if not os.path.isfile(directory+"/"+self._cfg.get("configfile")): # self._cfg.debug.stdout(f' --- No file {self._cfg.get("configfile")} in {directory}, skipping.', 1) # return {}, None # # # get config file for directory # dir_config = get_config(directory + "/" + self._cfg.get("configfile"), self._cfg) # if not dir_config: # self._cfg.debug.stdout(f' --- Cannot read config file {self._cfg.get("configfile")} in {directory}, ' # f'skipping.', 1) # return {}, None # # destdir = dir_config.get(section="Main", option="dest") # # destfiles is a dict of all files that will be created from the classes config # # key is the destination filename, values are all classes filenames that are used to # # build the file # destfiles = {} # try: # reverse_sort = dir_config.get(section="Main", option="reverse", boolean=True) # except ValueError: # reverse_sort = False # self._cfg.debug.stdout(f' +++ reverse_sort is {reverse_sort}', 3) # if os.access(directory + "/install.sh", os.X_OK): # subprocess.call([directory + "/install.sh"]) # # # walk through all know classes in directory and find filenames # for h in classes_for_host(reverse_sort): # # build classes directory # if h[0] != "": # classdir = directory+"/"+h[0]+"_"+h[1] # else: # classdir = directory+"/"+h[1] # self._cfg.debug.stdout(" ??? Looking for directory %s." % classdir, 4) # # if class directory exists # if os.path.isdir(classdir): # self._cfg.debug.stdout(" +++ Found directory %s, getting files." % classdir, 4) # # get list of files within this class directory # tempfiles = self.workconf(classdir) # self._cfg.debug.stdout(" +++ Got %d files: %s." % (len(tempfiles), str(tempfiles)), 4) # # put files into dict # for f in tempfiles: # destname = destdir+os.path.basename(f) # destination filename # if destname not in destfiles: # destfiles[destname] = [] # destfiles[destname].append(f) # append each file to dict # self._cfg.debug.stdout(f" +++ Added file to {destname}, now {len(destfiles[destname])} files: " # f"{destfiles[destname]}", 4) # # self._cfg.debug.stdout(" === workdir: %s, Files: %s" % (directory, str(destfiles)), 3) # self._cfg.debug.stdout(f" ================ done: workdir {directory} ===============", 1) # return destfiles, dir_config