From 34066bd4833d99399bde8f9a6fe5cf10833d01e0 Mon Sep 17 00:00:00 2001 From: simondgreenhill Date: Tue, 9 Apr 2019 17:07:20 -0500 Subject: [PATCH 1/5] Add QOS option to run command --- jrnr/jrnr.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/jrnr/jrnr.py b/jrnr/jrnr.py index 1a49b83..e337784 100644 --- a/jrnr/jrnr.py +++ b/jrnr/jrnr.py @@ -36,7 +36,7 @@ #SBATCH --account=co_laika # # QoS: -#SBATCH --qos=savio_lowprio +#SBATCH --qos={qos} # #SBATCH --nodes=1 # @@ -110,6 +110,7 @@ def _prep_slurm( filepath, jobname='slurm_job', partition='savio2', + qos='savio2_lowprio', job_spec=None, limit=None, uniqueid='"${SLURM_ARRAY_JOB_ID}"', @@ -161,6 +162,7 @@ def _prep_slurm( f.write(template.format( jobname=jobname, partition=partition, + qos=qos, numjobs=numjobs, jobs_per_node=jobs_per_node, maxnodes=(maxnodes-1), @@ -176,6 +178,7 @@ def run_slurm( filepath, jobname='slurm_job', partition='savio2', + qos='savio2_lowprio', job_spec=None, limit=None, uniqueid='"${SLURM_ARRAY_JOB_ID}"', @@ -189,6 +192,7 @@ def run_slurm( filepath=filepath, jobname=jobname, partition=partition, + qos=qos, job_spec=job_spec, limit=limit, uniqueid=uniqueid, @@ -363,6 +367,8 @@ def slurm(): '--jobname', '-j', default='test', help='name of the job') @click.option( '--partition', '-p', default='savio2', help='resource on which to run') + @click.option( + '--qos', '-q', default='savio2_lowprio', help='QOS for job') @click.option('--dependency', '-d', type=int, multiple=True) @click.option( '--logdir', '-L', default='log', help='Directory to write log files') @@ -375,6 +381,7 @@ def prep( jobname='slurm_job', dependency=None, partition='savio2', + qos='savio2_lowprio' maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): @@ -383,6 +390,7 @@ def prep( filepath=filepath, jobname=jobname, partition=partition, + qos=qos, job_spec=job_spec, jobs_per_node=jobs_per_node, maxnodes=maxnodes, @@ -405,6 +413,8 @@ def prep( '--jobname', '-j', default='test', help='name of the job') @click.option( '--partition', '-p', default='savio2', help='resource on which to run') + @click.option( + '--qos', '-q', default='savio2_lowprio', help='QOS for job') @click.option( '--dependency', '-d', type=int, multiple=True) @click.option( @@ -412,12 +422,14 @@ def prep( @click.option( '--uniqueid', '-u', default='"${SLURM_ARRAY_JOB_ID}"', help='Unique job pool id') + def run( limit=None, jobs_per_node=24, jobname='slurm_job', dependency=None, partition='savio2', + qos='savio2_lowprio', maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): @@ -429,6 +441,7 @@ def run( filepath=filepath, jobname=jobname, partition=partition, + qos=qos, job_spec=job_spec, jobs_per_node=jobs_per_node, maxnodes=maxnodes, @@ -441,6 +454,7 @@ def run( filepath=filepath, jobname=jobname+'_finish', partition=partition, + qos=qos, dependencies=('afterany', [slurm_id]), logdir=logdir, flags=['cleanup', slurm_id]) From 1b52d76a84ee9a599aa56efa1cd445e8933fb312 Mon Sep 17 00:00:00 2001 From: simondgreenhill Date: Tue, 9 Apr 2019 17:42:51 -0500 Subject: [PATCH 2/5] Change savio2_lowprio to savio_lowprio --- jrnr/jrnr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jrnr/jrnr.py b/jrnr/jrnr.py index e337784..958cc3d 100644 --- a/jrnr/jrnr.py +++ b/jrnr/jrnr.py @@ -110,7 +110,7 @@ def _prep_slurm( filepath, jobname='slurm_job', partition='savio2', - qos='savio2_lowprio', + qos='savio_lowprio', job_spec=None, limit=None, uniqueid='"${SLURM_ARRAY_JOB_ID}"', @@ -178,7 +178,7 @@ def run_slurm( filepath, jobname='slurm_job', partition='savio2', - qos='savio2_lowprio', + qos='savio_lowprio', job_spec=None, limit=None, uniqueid='"${SLURM_ARRAY_JOB_ID}"', @@ -368,7 +368,7 @@ def slurm(): @click.option( '--partition', '-p', default='savio2', help='resource on which to run') @click.option( - '--qos', '-q', default='savio2_lowprio', help='QOS for job') + '--qos', '-q', default='savio_lowprio', help='QOS for job') @click.option('--dependency', '-d', type=int, multiple=True) @click.option( '--logdir', '-L', default='log', help='Directory to write log files') @@ -381,7 +381,7 @@ def prep( jobname='slurm_job', dependency=None, partition='savio2', - qos='savio2_lowprio' + qos='savio_lowprio' maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): @@ -414,7 +414,7 @@ def prep( @click.option( '--partition', '-p', default='savio2', help='resource on which to run') @click.option( - '--qos', '-q', default='savio2_lowprio', help='QOS for job') + '--qos', '-q', default='savio_lowprio', help='QOS for job') @click.option( '--dependency', '-d', type=int, multiple=True) @click.option( @@ -429,7 +429,7 @@ def run( jobname='slurm_job', dependency=None, partition='savio2', - qos='savio2_lowprio', + qos='savio_lowprio', maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): From d344e4b9d89d178dfd99e60117daafa8edae205d Mon Sep 17 00:00:00 2001 From: sgreenhill Date: Tue, 9 Apr 2019 15:54:44 -0700 Subject: [PATCH 3/5] Fix small bug in jrnr.py --- jrnr/jrnr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jrnr/jrnr.py b/jrnr/jrnr.py index 958cc3d..1fbbe98 100644 --- a/jrnr/jrnr.py +++ b/jrnr/jrnr.py @@ -381,7 +381,7 @@ def prep( jobname='slurm_job', dependency=None, partition='savio2', - qos='savio_lowprio' + qos='savio_lowprio', maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): From c1e432cf3a9e11172424ecaf1daef1ddabc865df Mon Sep 17 00:00:00 2001 From: simondgreenhill Date: Fri, 19 Apr 2019 14:41:14 -0500 Subject: [PATCH 4/5] Update docs with --qos option --- docs/usage.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index f05004c..171a4b0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -101,6 +101,7 @@ Let's first have a look at the options with the run command. -d, --dependency INTEGER -L, --logdir TEXT Directory to write log files -u, --uniqueid TEXT Unique job pool id + -q, --qos TEXT QOS setting --help Show this message and exit. The most important options are ``u``, ``j`` and ``L``. To specify a job you need ``u`` and ``j`` since these parameters uniquely identify a job and allow you to track the progress of your jobs. An example command is below @@ -111,13 +112,13 @@ The most important options are ``u``, ``j`` and ``L``. To specify a job you need This creates a job with a unique id of `001` and a job name of `tas`. -By specifying some of the options listed above, you can adjust the behavior of your slurm jobs. For example, you can put your log files in a specific directory by specifying a value for argument ``L``. Additionally, if you want to use a specific partition on your cluster you can speify the `p` option. Similarly, if your job is particularly compute intensive, with ``n`` you can adjust the number of jobs per node. +By specifying some of the options listed above, you can adjust the behavior of your slurm jobs. For example, you can put your log files in a specific directory by specifying a value for argument ``L``. Additionally, if you want to use a specific partition on your cluster you can specify the `p` option. Similarly, if your job is particularly compute intensive, with ``n`` you can adjust the number of jobs per node. .. code-block:: bash $ python tas.py run -u 001 -j tas -L /logs/tas/ -p savio2_bigmem -n 10 -Its important to note that, by default, log files will be written to the directory where you are executing the file. Depending on how large your job is you may want to put these log files elsewhere. +It is important to note that, by default, log files will be written to the directory where you are executing the file. Depending on how large your job is you may want to put these log files elsewhere. If you want to fully take advantage of BRC's computing capacity you can run From 0b155cbefd552467c24e498e9760c0121f14aaa1 Mon Sep 17 00:00:00 2001 From: simondgreenhill Date: Fri, 19 Apr 2019 15:50:34 -0500 Subject: [PATCH 5/5] Small syntax changes for flake8 test --- jrnr/jrnr.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jrnr/jrnr.py b/jrnr/jrnr.py index 1fbbe98..cf4c7dc 100644 --- a/jrnr/jrnr.py +++ b/jrnr/jrnr.py @@ -381,7 +381,7 @@ def prep( jobname='slurm_job', dependency=None, partition='savio2', - qos='savio_lowprio', + qos='savio_lowprio', maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): @@ -433,7 +433,6 @@ def run( maxnodes=100, logdir='log', uniqueid='"${SLURM_ARRAY_JOB_ID}"'): - if not os.path.isdir(logdir): os.makedirs(logdir) @@ -454,7 +453,7 @@ def run( filepath=filepath, jobname=jobname+'_finish', partition=partition, - qos=qos, + qos=qos, dependencies=('afterany', [slurm_id]), logdir=logdir, flags=['cleanup', slurm_id]) @@ -548,7 +547,6 @@ def do_job(job_name, job_id, num_jobs=None, logdir='log'): run_job(**job_kwargs) except (KeyboardInterrupt, SystemExit): - try: logger.error('{} interupted, removing .lck file before exiting'.format(task_id)) os.remove(lock_file.format('lck'))