import argparse import json import sys import pprint from datetime import date, timedelta,datetime from library import add_time_options, get_time def seconds_to_string(secs): return str(timedelta(seconds=int(secs))) def is_active_job(job, start_date, end_date): job_time = job['update_time'] job_time = datetime.strptime(job_time, "%Y-%m-%dT%H:%M:%S.%f") if start_date <= job_time.date() <= (end_date + timedelta(days=1)): return True job_time = job['create_time'] job_time = datetime.strptime(job_time, "%Y-%m-%dT%H:%M:%S.%f") if start_date <= job_time.date() <= (end_date + timedelta(days=1)): return True return False def is_active_user(user, last, created, start_date, end_date): if start_date <= last.date() <= end_date + timedelta(days=1): return True if start_date <= created.date() <= end_date + timedelta(days=1): return True if 'jobs' not in user: return False if 'local' in user['jobs'] and len(user['jobs']['local']) > 0: for job in user['jobs']['local']: if is_active_job(job, start_date, end_date) is True: return True if 'slrum' in user['jobs'] and len(user['jobs']['slrum']) > 0: for job in user['jobs']['slurm']: if is_active_job(job, start_date, end_date) is True: return True return False def generate_user_info(users, output, start_date, end_date): user_list = [] for u in users: user = users[u] user_list.append(user) user_list = sorted(user_list, key=lambda x: x['id']) output.write('\nActive users in the selected period:\n') for user in user_list: last = datetime.strptime(user['update_time'],"%Y-%m-%dT%H:%M:%S.%f") created = datetime.strptime(user['create_time'],"%Y-%m-%dT%H:%M:%S.%f") if is_active_user(user, last, created, start_date, end_date) is False: continue output.write('--------------------------------------------\n') output.write('id: ' + str(user['id']) + "\n") output.write('email: ' + user['email'] + "\n") output.write('last login: ' + user['update_time'] + "\n") output.write('created on: ' + user['create_time'] + "\n") if start_date <= created.date() < end_date + timedelta(days=1): output.write("This is a new user!\n") if 'gold' in user: gold = user['gold'] for project in gold['total']: used = seconds_to_string(gold['total'][project]['used']) remaining= seconds_to_string(gold['total'][project]['remaining']) count = str(gold['total'][project]['count']) output.write('For project ' + project + " Used time: " + used + "("+ count +" jobs) Remaining time: " + remaining + "\n") if 'jobs' not in user: continue if 'local' in user['jobs']and len(user['jobs']['local']) > 0: output.write("Local Jobs:\n") tools = {} for job in user['jobs']['local']: if is_active_job(job, start_date, end_date) is False: continue if job['tool_id'] in tools: tool = tools[job['tool_id']] else: tool = {} tool['count'] = 0 tool['duration'] = 0 tools[job['tool_id']] = tool tool['count'] = tool['count'] + 1 tool['duration'] = tool['duration'] + int(job['duration']) for tool in tools: tool_data = tools[tool] used = seconds_to_string(tool_data['duration']) output.write('\tFor tool ' + tool + " Used time: " + used + "("+ str(tool_data['count']) +" jobs)\n") if 'slurm' in user['jobs'] and len(user['jobs']['slurm']) > 0: output.write("Remote Jobs:\n") projects = {} for job in user['jobs']['slurm']: if is_active_job(job, start_date, end_date) is False: continue if job['state'] == 'error': continue slurm_data = job['slurm'] project = slurm_data['project'] if project in projects: pdata = projects[project] else: pdata = {'tools': {},'slurm_time': 0, 'time':0, 'count' : 0, 'cpus' : 0} projects[project] = pdata pdata['count'] = pdata['count'] + 1 pdata['time'] = pdata['time'] + job['duration'] pdata['slurm_time'] = pdata['slurm_time'] + slurm_data['amount'] pdata['cpus'] = pdata['cpus'] + slurm_data['cpus'] if job['tool_id'] in pdata['tools']: tool = pdata['tools'][job['tool_id']] else: tool = {'slurm_time': 0, 'time':0, 'count' : 0, 'cpus' : 0} pdata['tools'][job['tool_id']] = tool tool['count'] = tool['count'] + 1 tool['time'] = tool['time'] + int(job['duration']) tool['slurm_time'] = tool['slurm_time'] + slurm_data['amount'] tool['cpus'] = tool['cpus'] + slurm_data['cpus'] for project in projects: pdata = projects[project] output.write("\t--- " + project + " ---\n") used = seconds_to_string(pdata['time']) slurm = seconds_to_string(pdata['slurm_time']) count = str(pdata['count']) cpus = str(pdata['cpus']) output.write("\t\tTotal slurm time: " + slurm + "("+ count +" jobs, "+ cpus +" cpus). Total job time: " + used + "\n") for tool_name in pdata['tools']: tool = pdata['tools'][tool_name] used = seconds_to_string(tool['time']) slurm = seconds_to_string(tool['slurm_time']) count = str(tool['count']) cpus = str(tool['cpus']) output.write('\t\tFor tool ' + tool_name + " slurm time: " + slurm + "(" + count + " jobs, " + cpus +" cpus). Job time: " + used + " \n") def init_tool_data(job, tools, user): tool_name = job['tool_id'] if tool_name in tools: tdata = tools[tool_name] else: tdata = {'jobs': 0, 'users': {}, 'local': {'jobs': 0, 'time': 0}, 'slurm': {'jobs': 0, 'time': 0, 'slurm_time': 0, 'cpus': 0}} tools[tool_name] = tdata user_name = user['email'] tdata['jobs'] = tdata['jobs'] + 1 if user_name in tdata['users']: tdata['users'][user_name] = tdata['users'][user_name] + 1 else: tdata['users'][user_name] = 1 return tdata def generate_tool_info(users, output, start_date, end_date, only_project=None): tools = {} user_list = [] for u in users: user = users[u] user_list.append(user) total = 0 time = 0 slurm_time = 0 user_list = sorted(user_list, key=lambda user: user['id']) for user in user_list: last = datetime.strptime(user['update_time'], "%Y-%m-%dT%H:%M:%S.%f") created = datetime.strptime(user['create_time'], "%Y-%m-%dT%H:%M:%S.%f") if is_active_user(user, last, created, start_date, end_date) is False: continue if 'jobs' not in user: continue if 'local' in user['jobs'] and len(user['jobs']['local']) > 0: for job in user['jobs']['local']: if is_active_job(job, start_date, end_date) is False: continue total = total + 1 time = time + job['duration'] tdata = init_tool_data(job, tools, user) tdata['local']['jobs'] = tdata['local']['jobs'] + 1 tdata['local']['time'] = tdata['local']['time'] + job['duration'] if 'slurm' in user['jobs'] and len(user['jobs']['slurm']) > 0: for job in user['jobs']['slurm']: if is_active_job(job, start_date, end_date) is False: continue if only_project is not None and job['slurm']['project'] != only_project: continue total = total + 1 time = time + job['duration'] slurm_time = slurm_time + job['slurm']['amount'] tdata = init_tool_data(job, tools, user) tdata['slurm']['jobs'] = tdata['slurm']['jobs'] + 1 tdata['slurm']['time'] = tdata['slurm']['time'] + job['duration'] tdata['slurm']['slurm_time'] = tdata['slurm']['slurm_time'] + job['slurm']['amount'] tdata['slurm']['cpus'] = tdata['slurm']['cpus'] + job['slurm']['cpus'] output.write('\nActive tools in the selected period:\n') output.write("\tTotal Jobs : " + str(total) +"\n") output.write("\tTotal time : " + seconds_to_string(time) + "\n") output.write("\tTotal slurm time : " + seconds_to_string(slurm_time) + "\n") for tool in tools: tool_data = tools[tool] output.write('--------------------------------------------\n') output.write('tool: ' + tool + "\n") output.write('jobs: ' + str(tool_data['jobs']) + "\n") output.write('users: ' + str(len(tool_data['users'])) + "\n") if tool_data['local']['jobs'] > 0 and only_project is None: output.write("Local Jobs:\n") output.write('\tjobs: ' + str(tool_data['local']['jobs']) + "\n") output.write('\ttime: ' + seconds_to_string(tool_data['local']['time']) + "\n") if tool_data['slurm']['jobs'] > 0: output.write("Remote Jobs:\n") output.write('\tjobs: ' + str(tool_data['slurm']['jobs']) + "\n") output.write('\ttime: ' + seconds_to_string(tool_data['slurm']['time']) + "\n") output.write('\tslurm time: ' + seconds_to_string(tool_data['slurm']['slurm_time']) + "\n") output.write('\tcpus: ' + str(tool_data['slurm']['cpus']) + "\n") if __name__ == "__main__": parser = argparse.ArgumentParser(prog='summary script') add_time_options(parser) parser.add_argument('--users', action='store_true', help='Generate users info') parser.add_argument('--tools', action='store_true', help='Generate tools info') parser.add_argument('--project', nargs=1, help='Information for a particular project') parser.add_argument('--output', nargs=1, help="Output file") parser.add_argument('file', help='Portal json report file file') args = parser.parse_args() if hasattr(args, 'help') and args.help is not None: parser.print_help() sys.exit(0) print "Reading data from: " + args.file with open(args.file) as json_data: d = json.load(json_data) output = sys.stdout if args.output is not None: output = open(args.output[0], 'w') start = datetime.strptime(d['start'], "%Y-%m-%d").date() end = datetime.strptime(d['end'], "%Y-%m-%d").date() start, end = get_time(args, def_start=start, def_end=end) output.write("Report from: " + start.isoformat() + " to: " + end.isoformat() + "\n") if args.users is True: generate_user_info(d['data'], output, start, end) project = None if hasattr(args, 'project'): project = args.project if project is not None and len(project) == 0: project = None if project is not None: project = project[0] if args.tools is True: generate_tool_info(d['data'], output, start, end, only_project=project) if args.output is not None: output.close()