#!/u01/app/python/current_version/bin/python3 import requests import json import os import shutil import subprocess import zipfile import logging import argparse import string import random import smtplib from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate from email import encoders from requests.packages.urllib3.exceptions import InsecureRequestWarning # FUNCTIONS ########### def set_constants(): url_base ='https://reman.arval.com/rest/api/2/' auth = ('dbjira', 'marcguillard') return (url_base, auth) def parse_command_line_args(): parser = argparse.ArgumentParser(description='Toool for deploy JIRA Oracle scripts') parser.add_argument('-j','--jira',help='JIRA task (ex: JIRA-12345)', required=True) parser.add_argument('-i','--ignorestatus', action='store_true', default=False ,help='Ignore JIRA status (force execution for JIRA task in BUILD/CLOSED status)', required=False) args = parser.parse_args() jira=args.jira.upper() ignorestatus=args.ignorestatus return (jira, ignorestatus) def start_logging(logfile): logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # create a file handler handler = logging.FileHandler(logfile) handler.setLevel(logging.INFO) # create a logging format formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) # add the handlers to the logger logger.addHandler(handler) return logger def get_task_json(url_base, auth, jira_task): url_end='issue/'+jira_task r = requests.get(url_base+url_end, auth=auth, verify=False) jira_json = r.json() return jira_json def dos2unix(filename): content = '' outsize = 0 with open(filename, 'rb') as infile: content = infile.read() with open(filename, 'wb') as output: for line in content.splitlines(): outsize += len(line) + 1 output.write(line + b'\n') return def generate_password(): password = 'D#0'+''.join(random.choices(string.ascii_uppercase + string.digits, k=9)) return password def create_proxy_user(main_sql_file): fileContents = open(main_sql_file,"r").read() for line in fileContents: if 'DEFINE password_' in line: database_username = line.split('DEFINE password_')[0] print (database_username) return def generate_local_deploy_package(url_base, auth, jira_task): url_end='issue/'+jira_task r = requests.get(url_base+url_end, auth=auth, verify=False) jira_json = r.json() jira_nfs_scripts_path=jira_json['fields']['customfield_12084'] if 'dsi_mer' in jira_nfs_scripts_path: jira_local_scripts_path=jira_nfs_scripts_path.replace('file://Frshares0105.france.intra.corp/dsi_mer','/mnt/jira/MER') elif 'dsi_rtpp' in jira_nfs_scripts_path: jira_local_scripts_path=jira_nfs_scripts_path.replace('file://Frshares0105.france.intra.corp/dsi_rtpp','/mnt/jira/RTPP') else: jira_local_scripts_path=jira_nfs_scripts_path.replace('file://Frshares0105.france.intra.corp/dsi_rtp','/mnt/jira/RTP') # Create local jira package jira_target_database=jira_json['fields']['customfield_12277']['child']['value'].strip() jira_local_package_dir = os.path.dirname(os.path.abspath(__file__)) + '/packages_jira/' + jira_task + '/' + jira_target_database os.makedirs(jira_local_package_dir, mode=0o755, exist_ok=True) # Copy files from NFS into local directory # and generate a new "run all" sql script for manage connection strings using a proxy user main_sql_file = jira_local_package_dir + '/' + jira_target_database +'.sql' src_files = os.listdir(jira_local_scripts_path) for file_name in src_files: full_file_name = os.path.join(jira_local_scripts_path, file_name) if (os.path.isfile(full_file_name)): if (jira_target_database + '.sql') in file_name: shutil.copy(full_file_name, main_sql_file) os.chmod(main_sql_file, 0o755) dos2unix(main_sql_file) else: shutil.copy(full_file_name, jira_local_package_dir) os.chmod(jira_local_package_dir + '/' + file_name, 0o755) dos2unix(jira_local_package_dir + '/' + file_name) contents_of_create_deploy_user_sql=[] contents_of_drop_deploy_user_sql=[] dba_username = 'system' dba_password = 'plusdacces' deploy_username = 'DEPLOY' deploy_password = generate_password() contents_of_create_deploy_user_sql.append('connect ' + dba_username + '/' + dba_password + '@&INSTANCE') contents_of_create_deploy_user_sql.append('create user ' + deploy_username + ' identified externally;') contents_of_create_deploy_user_sql.append('alter user ' + deploy_username + ' identified by ' + deploy_password + ';') contents_of_create_deploy_user_sql.append('grant create session to ' + deploy_username + ';') contents_of_main_sql_file = open(main_sql_file,'r', encoding='utf-8', errors='replace').readlines() for line in contents_of_main_sql_file: line=line.rstrip() line.replace(' ', '') if 'DEFINE password_' in line: database_username = line.split('DEFINE password_')[1] database_username = database_username.split('=')[0] contents_of_create_deploy_user_sql.append('alter user ' + database_username + ' grant connect through ' + deploy_username + ';') contents_of_create_deploy_user_sql.append('disconnect') contents_of_drop_deploy_user_sql.append('connect ' + dba_username + '/' + dba_password + '@&INSTANCE') contents_of_drop_deploy_user_sql.append('drop user ' + deploy_username + ';') contents_of_drop_deploy_user_sql.append('disconnect') contents_of_new_sql_file = [] for line in contents_of_main_sql_file: if 'DEFINE INSTANCE' in line.upper(): contents_of_new_sql_file.append(line + '\n\n') contents_of_new_sql_file.append('-- create deploy user \n') # Add create deploy user section for line_deploy_user in contents_of_create_deploy_user_sql: contents_of_new_sql_file.append(line_deploy_user + '\n') elif 'DEFINE PASSWORD' in line.upper(): # skip the line pass elif line.startswith('SET logPath'): # for the PRODUCTION only # skip the line pass elif line.startswith('spool %logPath%'): # for the PRODUCTION only # change the spool file name line_stripped =line.rstrip() words = line_stripped.split('\\') new_spool_name = jira_local_package_dir + '/' + words[-1] contents_of_new_sql_file.append('spool '+ new_spool_name + '\n') pass elif 'exit' in line: # skip the line pass elif line.upper().startswith('CONNECT'): database_user = line.split('/')[0].split(' ')[1] contents_of_new_sql_file.append('connect ' + deploy_username + '[' + database_user +']'+ '/' + deploy_password + '@&INSTANCE' + '\n') else: contents_of_new_sql_file.append(line) contents_of_new_sql_file.append('\n-- drop deploy user \n') for line_deploy_user in contents_of_drop_deploy_user_sql: contents_of_new_sql_file.append(line_deploy_user + '\n') contents_of_new_sql_file.append('exit') contents_of_new_sql_file.append('\n\n') f = open(main_sql_file, 'w') contents_of_new_sql_file="".join(contents_of_new_sql_file) f.write(contents_of_new_sql_file) f.close() return (jira_local_package_dir, main_sql_file) def execute_sql_file(directory, sql_file, nls_lang_command): shellcommand='export TNS_ADMIN=/u01/app/oracle/admin/JIRA; ' + nls_lang_command + ';' + ' cd ' + directory + '; '+ 'sqlplus /nolog @'+sql_file try: cmd = subprocess.run( shellcommand, check=True, shell=True, stdout=subprocess.PIPE, ) except subprocess.CalledProcessError as err: pass return def zip_logfiles(directory, zipname): fantasy_zip = zipfile.ZipFile(directory + '/'+ zipname +'.zip', 'w') for folder, subfolders, files in os.walk(directory): for file in files: if file.endswith('.log'): fantasy_zip.write(os.path.join(folder, file), os.path.relpath(os.path.join(folder,file), directory), compress_type = zipfile.ZIP_DEFLATED) fantasy_zip.close() return def upload_zip_logfiles_to_jira(url_base, auth, jira_task, file): url_end='issue/'+ jira_task + '/attachments' headers = {"X-Atlassian-Token": "nocheck"} files = {'file': open(file, 'rb')} r = requests.post(url_base+url_end, auth=auth, verify=False, headers=headers, files=files) return def jira_transition(url_base, auth, jira_task, transion_target): url_end = 'issue/' + jira_task + '/transitions?expand=transitions.fields' headers = { 'Content-Type': 'application/json', } if transion_target == 'BUILD': # OPEN to BUILD transition data = json.dumps({ "update": { "comment": [ { "add": { "body": "Task started, work in progress." } } ] }, "transition": { "id": "41" } }) elif transion_target == 'CLOSE': # BUILD to CLOSED transition data = json.dumps({ "update": { "comment": [ { "add": { "body": "Task done." } } ] }, "transition": { "id": "31" }, "fields": { "resolution": { "name": "Fixed" } } }) # Make the JIRA transition r = requests.post(url_base+url_end, data=data, auth=auth, verify=False ,headers=headers) return def count_errors_in_logfile(logfile, error_pattern): logfile_contents = open(logfile, 'r', encoding='utf-8', errors='replace').readlines() error_count = 0 for line in logfile_contents: if line.startswith (error_pattern): error_count = error_count + 1 return error_count def count_errors_in_directory(directory, error_pattern): # Return all errors as a dictionary all_errors={} for folder, subfolders, files in os.walk(directory): for file in files: if file.endswith('.log'): error_count = count_errors_in_logfile(os.path.join(folder, file), error_pattern) all_errors[file] = error_count return all_errors def count_warnings_in_logfile(logfile, warning_pattern_start): logfile_contents = open(logfile, 'r', encoding='utf-8', errors='replace').readlines() warning_count = 0 for line in logfile_contents: if line.startswith (warning_pattern_start): warning_count = warning_count + 1 return warning_count def count_warnings_in_directory(directory, warning_pattern_start): # Return all warnings as a dictionary all_warnings={} for folder, subfolders, files in os.walk(directory): for file in files: if file.endswith('.log'): warnings_count = count_warnings_in_logfile(os.path.join(folder, file), warning_pattern_start) all_warnings[file] = warnings_count return all_warnings def comment_jira(url_base, auth, jira_task, comment): url_end='issue/'+ jira_task + '/comment' headers = { 'Content-Type': 'application/json', } data = json.dumps({ 'body':comment }) r = requests.post(url_base+url_end, data=data, auth=auth, verify=False ,headers=headers) return def send_mail(send_from, send_to, subject, text, files=[], server="localhost"): assert type(send_to)==list assert type(files)==list msg = MIMEMultipart() msg['From'] = send_from msg['To'] = COMMASPACE.join(send_to) msg['Date'] = formatdate(localtime=True) msg['Subject'] = subject msg.attach( MIMEText(text) ) for f in files: part = MIMEBase('application', "octet-stream") part.set_payload( open(f,"rb").read() ) encoders.encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(f)) msg.attach(part) smtp = smtplib.SMTP(server) smtp.sendmail(send_from, send_to, msg.as_string()) smtp.close() return def get_watchers(url_base, auth, jira_task): # Get the JIRA watchers email list watchers_list = [] url_end='issue/'+ jira_task + '/watchers' r = requests.get(url_base+url_end, auth=auth, verify=False) for watcher in r.json()['watchers']: watchers_list.append(watcher['emailAddress']) return watchers_list def create_connect_test_script(jira_local_package_dir, main_sql_file): contents_of_connect_test_script = [] main_sql_file_contents = open(main_sql_file, 'r', encoding='utf-8', errors='replace').readlines() line_number = -1 for line in main_sql_file_contents: line_number +=1 line_stripped =line.rstrip() if line_stripped.startswith("@"): contents_of_connect_test_script.append("select * from dual;\n") elif line_stripped.startswith("spool"): # supress intermediate spool commands pass elif line_stripped.startswith("disconnect"): prev_line_stripped =main_sql_file_contents[line_number - 1].rstrip() if prev_line_stripped.startswith("alter user"): # add spool after disconnect contents_of_connect_test_script.append(line) contents_of_connect_test_script.append("\nspool connect_test.txt\n") elif prev_line_stripped.startswith("drop user"): # add a line to extract NLS_LANG value contents_of_connect_test_script.append("select 'export NLS_LANG=.'|| value SQLPLUS_NLS from nls_database_parameters where parameter='NLS_CHARACTERSET';\n") # add spool off and echo before drop deploy user contents_of_connect_test_script.append("spool off\n") contents_of_connect_test_script.append(line) else: contents_of_connect_test_script.append(line) connect_test_script = jira_local_package_dir + "/connect_test.sql" f = open(connect_test_script, 'w') contents_of_connect_test_script="".join(contents_of_connect_test_script) f.write(contents_of_connect_test_script) f.close() return connect_test_script def get_nls_lang_command(jira_local_package_dir): connect_test_script_logfile = jira_local_package_dir + "/connect_test.txt" contents_of_connect_test_script = open(connect_test_script_logfile, 'r', encoding='utf-8', errors='replace').readlines() for line in contents_of_connect_test_script: line_stripped = line.rstrip() if line_stripped.startswith("export NLS_LANG"): nls_lang_command = line_stripped return nls_lang_command # MAIN ########### (jira_task, ignorestatus) = parse_command_line_args() script_path = os.path.dirname(os.path.abspath(__file__)) script_name = os.path.basename(__file__) requests.packages.urllib3.disable_warnings(InsecureRequestWarning) (url_base, auth) = set_constants() global logger global jira_json jira_json = get_task_json(url_base, auth, jira_task) jira_description = jira_json['fields']['description'] jira_statusid = jira_json['fields']['status']['id'] jira_statusname = jira_json['fields']['status']['name'] jira_target_database = jira_json['fields']['customfield_12277']['child']['value'] jira_nfs_scripts_path = jira_json['fields']['customfield_12084'] jira_comment = jira_json['fields']['comment']['comments'] jira_reporter = jira_json['fields']['reporter']['emailAddress'] if jira_statusname == 'Build' or jira_statusname == 'Closed' or jira_statusname == 'Frozen': if ignorestatus == False: print (jira_task + ' is in ' + jira_statusname.upper() + ' status') print ('Use -i|--ignorestatus to force the execution' ) exit (0) logger = start_logging(script_path+'/logs/jira_deploy.log') logger.info (jira_task + ' : ' + 'STARTED') logger.info (jira_task + ' : status is ' + jira_statusname) logger.info (jira_task + ' : target database is ' + jira_target_database) logger.info (jira_task + ' : NFS scripts path is ' + jira_nfs_scripts_path) for user_comment in jira_comment: logger.info (jira_task + ' : user comment is '+user_comment['body']) jira_transition(url_base, auth, jira_task, 'BUILD') comment_jira(url_base, auth, jira_task, 'Execution in progress.') logger.info (jira_task + ' : ' + 'status transition to BUILD') # Download JIRA packages and generate the main sql file (jira_local_package_dir, main_sql_file) = generate_local_deploy_package(url_base, auth, jira_task) logger.info (jira_task + ' : ' + 'local package copied under ' + jira_local_package_dir) logger.info (jira_task + ' : ' + 'main SQL file generated in ' + main_sql_file) # Before the execution of the main sql file, we check the database connectivity of deploy user through the differents schemas connect_test_script = create_connect_test_script(jira_local_package_dir, main_sql_file) connect_test_script_logfile = connect_test_script.replace('.sql', '.txt') logger.info (jira_task + ' : ' + 'connexion test SQL file generated in ' + connect_test_script) logger.info (jira_task + ' : ' + 'execution of connexion test SQL file started') execute_sql_file(jira_local_package_dir, connect_test_script, 'export NLS_LANG=.UTF8') logger.info (jira_task + ' : ' + 'execution of connexion test SQL file finished') # check for ORA- and SP2-0640: Not connected in logfile ora_errors = count_errors_in_logfile(connect_test_script_logfile, 'ORA-') sp2_not_connected_errors = count_errors_in_logfile(connect_test_script_logfile, 'SP2-0640: Not connected') if (ora_errors + sp2_not_connected_errors) > 0: logger.error (jira_task + ' : ' + str(ora_errors + sp2_not_connected_errors) + ' error(s) in ' + connect_test_script_logfile) print ("Database connexion issue, please check " + connect_test_script_logfile) exit (1) # Execution of the main sql file nls_lang_command = get_nls_lang_command(jira_local_package_dir) logger.info (jira_task + ' : ' + 'execution of main SQL file started') execute_sql_file(jira_local_package_dir, main_sql_file, nls_lang_command) logger.info (jira_task + ' : ' + 'execution of main SQL file finished') zip_logfiles(jira_local_package_dir ,jira_task) logger.info (jira_task + ' : ' + 'zip logfile(s)') upload_zip_logfiles_to_jira(url_base, auth, jira_task, jira_local_package_dir+'/'+jira_task+'.zip') logger.info (jira_task + ' : ' + 'upload zipped logfile(s) to JIRA') # check for ORA- in logfiles ora_errors = count_errors_in_directory (jira_local_package_dir, 'ORA-') count_all_errors = 0 for logfile, error_count in ora_errors.items(): count_all_errors = count_all_errors + error_count if error_count > 0: logger.info (jira_task + ' : ' + logfile + ' has ' + str(error_count) + ' ORA- error message(s)') else: logger.info (jira_task + ' : ' + logfile + ' has no error messages') # check for SP2-0640: Not connected in logfiles sp2_not_connected_errors = count_errors_in_directory (jira_local_package_dir, 'SP2-0640: Not connected') count_all_sp2_not_connected_errors = 0 for logfile, sp2_count in sp2_not_connected_errors.items(): count_all_sp2_not_connected_errors = count_all_sp2_not_connected_errors + sp2_count if error_count > 0: logger.info (jira_task + ' : ' + logfile + ' has ' + str(sp2_count) + ' SP2-0640: Not connected error message(s)') else: logger.info (jira_task + ' : ' + logfile + ' has no messages') # check for SP2-0042: unknown command in logfiles sp2_unknown_command_errors = count_errors_in_directory (jira_local_package_dir, 'SP2-0042: unknown command') count_all_sp2_unknown_command_errors = 0 for logfile, sp2_0042_count in sp2_unknown_command_errors.items(): count_all_sp2_unknown_command_errors = count_all_sp2_unknown_command_errors + sp2_0042_count if error_count > 0: logger.info (jira_task + ' : ' + logfile + ' has ' + str(sp2_0042_count) + ' SP2-0042: unknown command error message(s)') else: logger.info (jira_task + ' : ' + logfile + ' has no messages') # check for SP2-0734: unknown command in logfiles sp2_unknown_command_2_errors = count_errors_in_directory (jira_local_package_dir, 'SP2-0734: unknown command beginning') count_all_sp2_unknown_command_2_errors = 0 for logfile, sp2_0734_count in sp2_unknown_command_2_errors.items(): count_all_sp2_unknown_command_2_errors = count_all_sp2_unknown_command_2_errors + sp2_0734_count if error_count > 0: logger.info (jira_task + ' : ' + logfile + ' has ' + str(sp2_0734_count) + ' SP2-0734: unknown command beginning error message(s)') else: logger.info (jira_task + ' : ' + logfile + ' has no messages') # check for Warning: ... created with compilation errors. in logfiles ora_warnings = count_warnings_in_directory (jira_local_package_dir, 'Warning:') count_all_warnings = 0 for logfile, warnings_count in ora_warnings.items(): count_all_warnings = count_all_warnings + warnings_count if warnings_count > 0: logger.info (jira_task + ' : ' + logfile + ' has ' + str(warnings_count) + ' warning message(s)') else: logger.info (jira_task + ' : ' + logfile + ' has no warning messages') email_text = 'Hello,\n\n' error_summary = '' sp2_not_connected_summary = '' warning_summary = '' if (count_all_errors + count_all_sp2_not_connected_errors + count_all_sp2_unknown_command_errors + count_all_sp2_unknown_command_2_errors + count_all_warnings) == 0: comment_jira(url_base, auth, jira_task, 'Done without errors.') jira_transition(url_base, auth, jira_task, 'CLOSE') logger.info (jira_task + ' : ' + 'status transition to CLOSE') email_subject = jira_target_database + ' : ' + jira_task + ' has been executed executed without errors.' email_text = email_text + 'Jira task ' + jira_task + ' has been executed executed without errors.\n' email_text = email_text + 'Log files enclosed.' else: for element in ora_errors: if ora_errors[element] > 0: error_summary = error_summary + '\nLogfile ' + element + ' has ' + str(ora_errors[element]) + ' error(s)' for element in sp2_not_connected_errors: if sp2_not_connected_errors[element] > 0: error_summary = error_summary + '\nLogfile ' + element + ' has ' + str(sp2_not_connected_errors[element]) + ' error(s)' for element in sp2_unknown_command_errors: if sp2_unknown_command_errors[element] > 0: error_summary = error_summary + '\nLogfile ' + element + ' has ' + str(sp2_unknown_command_errors[element]) + ' error(s)' for element in sp2_unknown_command_2_errors: if sp2_unknown_command_2_errors[element] > 0: error_summary = error_summary + '\nLogfile ' + element + ' has ' + str(sp2_unknown_command_2_errors[element]) + ' error(s)' for element in ora_warnings: if ora_warnings[element] > 0: warning_summary = warning_summary + '\nLogfile ' + element + ' has ' + str(ora_warnings[element]) + ' warning(s)' comment_jira(url_base, auth, jira_task, 'Done with ' + str(count_all_errors + count_all_sp2_not_connected_errors + count_all_sp2_unknown_command_errors + count_all_sp2_unknown_command_2_errors) + ' error(s) and ' + str(count_all_warnings) + ' warning(s), please check the enclosed logfile(s):\n') if error_summary != '': comment_jira(url_base, auth, jira_task, 'Errors summary: \n' + error_summary) if sp2_not_connected_summary != '': comment_jira(url_base, auth, jira_task, 'SP2-0640: Not connected summary: \n' + sp2_not_connected_summary) if warning_summary != '': comment_jira(url_base, auth, jira_task, 'Warnings summary: \n' + warning_summary) email_subject = jira_target_database + ' : ' + jira_task + ' executed with ' + str(count_all_errors + count_all_sp2_not_connected_errors + count_all_sp2_unknown_command_errors + count_all_sp2_unknown_command_2_errors + count_all_warnings) + ' error(s)/warning(s).' email_text = email_text + jira_task + ' has been executed with ' + str(count_all_errors + count_all_sp2_not_connected_errors + count_all_sp2_unknown_command_errors + count_all_sp2_unknown_command_2_errors + count_all_warnings) + ' errors/warnings:\n\n' + '\n\n'.join(error_summary.split('\n')) + '\n\n'.join(sp2_not_connected_summary.split('\n')) + '\n\n' + '\n\n'.join(warning_summary.split('\n')) + '\n\nPlease ckeck the logfiles and close the JIRA.' # Send an email to oracle, release, JIRA reporter and all JIRA watchers send_to = ['dbm@arval.fr','release@arval.fr'] send_to.append(jira_reporter) watchers_list = get_watchers(url_base, auth, jira_task) send_to.extend(watchers_list) files=[] files.append(jira_local_package_dir+'/'+jira_task+'.zip') send_mail(send_from = 'no-reply@arval.com', send_to = send_to, subject = email_subject, text = email_text, files=files, server="localhost") logger.info (jira_task + ' : ' + 'FINISHED')