Class: OodCore::Job::Adapters::Slurm

Inherits:
OodCore::Job::Adapter show all
Defined in:
lib/ood_core/job/adapters/slurm.rb

Overview

An adapter object that describes the communication with a Slurm resource manager for job management.

Defined Under Namespace

Classes: Batch

Constant Summary collapse

STATE_MAP =

Mapping of state codes for Slurm

{
  'BF' => :completed,  # BOOT_FAIL
  'CA' => :completed,  # CANCELLED
  'CD' => :completed,  # COMPLETED
  'CF' => :queued,     # CONFIGURING
  'CG' => :running,    # COMPLETING
  'F'  => :completed,  # FAILED
  'NF' => :completed,  # NODE_FAIL
  'PD' => :queued,     # PENDING
  'PR' => :suspended,  # PREEMPTED
  'RV' => :completed,  # REVOKED
  'R'  => :running,    # RUNNING
  'SE' => :completed,  # SPECIAL_EXIT
  'ST' => :running,    # STOPPED
  'S'  => :suspended,  # SUSPENDED
  'TO' => :completed,  # TIMEOUT
  'OOM' => :completed, # OUT_OF_MEMORY

  'BOOT_FAIL'     => :completed,
  'CANCELED'      => :completed,
  'COMPLETED'     => :completed,
  'DEADLINE'      => :completed,
  'FAILED'        => :completed,
  'NODE_FAIL'     => :completed,
  'OUT_OF_MEMORY' => :completed,
  'PENDING'       => :queued,
  'PREEMPTED'     => :completed,
  'RUNNING'       => :running,
  'REQUEUED'      => :queued,
  'REVOKED'       => :completed,
  'SUSPENDED'     => :suspended,
  'TIMEOUT'       => :completed,
}

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from OodCore::Job::Adapter

#info_all_each, #info_where_owner_each, #job_name_illegal_chars, #sanitize_job_name, #supports_job_arrays?

Constructor Details

#initialize(opts = {}) ⇒ Slurm

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Returns a new instance of Slurm.

Parameters:

  • opts (#to_h) (defaults to: {}) —

    the options defining this adapter

Options Hash (opts):

  • :slurm (Batch) —

    The Slurm batch object

See Also:



599
600
601
602
603
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 599

def initialize(opts = {})
  o = opts.to_h.symbolize_keys

  @slurm = o.fetch(:slurm) { raise ArgumentError, "No slurm object specified. Missing argument: slurm" }
end

Class Method Details

.gpus_from_gres(gres) ⇒ Integer

Get integer representing the number of gpus used by a node or job, calculated from gres string

Returns:

  • (Integer) —

    the number of gpus in gres



49
50
51
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 49

def self.gpus_from_gres(gres)
  gres.to_s.scan(/gpu[^(,]*[:=](\d+)/).flatten.map(&:to_i).sum
end

Instance Method Details

#accounts ⇒ Array<String>

Retrieve the accounts available to use for the current user.

Returns:

  • (Array<String>) —

    the accounts available to the user.



695
696
697
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 695

def accounts
  @slurm.accounts
end

#cluster_info ⇒ Hash

Retrieve info about active and total cpus, gpus, and nodes

Returns:

  • (Hash) —

    information about cluster usage



688
689
690
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 688

def cluster_info
  @slurm.get_cluster_info
end

#delete(id) ⇒ void

This method returns an undefined value.

Delete the submitted job

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



849
850
851
852
853
854
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 849

def delete(id)
  @slurm.delete_job(id.to_s)
rescue Batch::Error => e
  # assume successful job deletion if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#directive_prefix ⇒ Object 



856
857
858
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 856

def directive_prefix
  '#SBATCH'
end

#hold(id) ⇒ void

This method returns an undefined value.

Put the submitted job on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



825
826
827
828
829
830
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 825

def hold(id)
  @slurm.hold_job(id.to_s)
rescue Batch::Error => e
  # assume successful job hold if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#info(id) ⇒ Info

Retrieve job info from the resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

  • (Info) —

    information describing submitted job

Raises:

See Also:



755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 755

def info(id)
  id = id.to_s
  info_ary = @slurm.get_jobs(id: id).map do |v|
    parse_job_info(v)
  end

  # If no job was found we assume that it has completed
  info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Info.new(
      id: id,
      status: :completed
    )
  else
    raise JobAdapterError, e.message
  end
end

#info_all(attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs from the resource manager

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:

See Also:



703
704
705
706
707
708
709
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 703

def info_all(attrs: nil)
  @slurm.get_jobs(attrs: attrs).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#info_historic(opts: {}) ⇒ Array<Info>

Retrieve historic info for all completed jobs from the resource manager.

Known options: job_ids [Array<#to_s>] optional list of job ids to filter the results. states [Array<#to_s>] optional list of job state codes. Selects jobs based on their state during the time period given. from [#to_s] optional date string to filter jobs in any state after the specified time. If states are provided, filter jobs in these states after this period to [#to_s] optional date string to filter jobs in any state before the specified time. If states are provided, filter jobs in these states before this period. show_steps [#Boolean] optional boolean to filter job steps from the results.

Returns:

  • (Array<Info>) —

    information describing submitted jobs

See Also:



725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 725

def info_historic(opts: {})
  job_ids    = opts.fetch(:job_ids, [])
  states     = opts.fetch(:states, [])
  from       = opts.fetch(:from, nil)
  to         = opts.fetch(:to, nil)
  show_steps = opts.fetch(:show_steps, false)
  @slurm.sacct_info(job_ids, states, from, to, show_steps).map do |v|
    Info.new(
      id: v[:job_id],
      status: get_state(v[:state]),
      job_name: v[:job_name],
      job_owner: v[:user],
      procs: v[:alloc_cpus],
      queue_name: v[:partition],
      wallclock_time: duration_in_seconds(v[:elapsed]),
      wallclock_limit: duration_in_seconds(v[:time_limit]),
      cpu_time: duration_in_seconds(v[:total_cpu]),
      submission_time: parse_time(v[:submit_time]),
      dispatch_time: parse_time(v[:start_time]),
      native: v,
      gpus: self.class.gpus_from_gres(v[:gres])
    )
  end
end

#info_where_owner(owner, attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs for a given owner or owners from the resource manager

Parameters:

  • owner (#to_s, Array<#to_s>) —

    the owner(s) of the jobs

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:



780
781
782
783
784
785
786
787
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 780

def info_where_owner(owner, attrs: nil)
  owner = Array.wrap(owner).map(&:to_s).join(',')
  @slurm.get_jobs(owner: owner).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#nodes ⇒ Object 



864
865
866
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 864

def nodes
  @slurm.nodes
end

#queues ⇒ Object 



860
861
862
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 860

def queues
  @slurm.queues
end

#release(id) ⇒ void

This method returns an undefined value.

Release the job that is on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



837
838
839
840
841
842
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 837

def release(id)
  @slurm.release_job(id.to_s)
rescue Batch::Error => e
  # assume successful job release if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#status(id) ⇒ Status

Retrieve job status from resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

Raises:

See Also:



794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 794

def status(id)
  id = id.to_s
  jobs = @slurm.get_jobs(
    id: id,
    attrs: [:job_id, :array_job_task_id, :state_compact]
  )
  # A job id can return multiple jobs if it corresponds to a job array
  # id, so we need to find the job that corresponds to the given job id
  # (if we can't find it, we assume it has completed)
  #
  # Match against the job id or the formatted job & task id "1234_0"
  if job = jobs.detect { |j| j[:job_id] == id || j[:array_job_task_id] == id }
    Status.new(state: get_state(job[:state_compact]))
  else
    # set completed status if can't find job id
    Status.new(state: :completed)
  end
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Status.new(state: :completed)
  else
    raise JobAdapterError, e.message
  end
end

#submit(script, after: [], afterok: [], afternotok: [], afterany: []) ⇒ String

Submit a job with the attributes defined in the job template instance

Parameters:

  • script (Script) —

    script object that describes the script and attributes for the submitted job

  • after (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution at any point after dependent jobs have started execution

  • afterok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with no errors

  • afternotok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with errors

  • afterany (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution after dependent jobs have terminated

Returns:

  • (String) —

    the job id returned after successfully submitting a job

Raises:

See Also:



620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 620

def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
  after      = Array(after).map(&:to_s)
  afterok    = Array(afterok).map(&:to_s)
  afternotok = Array(afternotok).map(&:to_s)
  afterany   = Array(afterany).map(&:to_s)

  # Set sbatch options
  args = []
  # ignore args, don't know how to do this for slurm
  args.concat ["-H"] if script.submit_as_hold
  args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
  args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
  args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
  if script.email_on_started && script.email_on_terminated
    args.concat ["--mail-type", "ALL"]
  elsif script.email_on_started
    args.concat ["--mail-type", "BEGIN"]
  elsif script.email_on_terminated
    args.concat ["--mail-type", "END"]
  elsif script.email_on_started == false && script.email_on_terminated == false
    args.concat ["--mail-type", "NONE"]
  end
  args.concat ["-J", script.job_name] unless script.job_name.nil?
  args.concat ["-i", script.input_path] unless script.input_path.nil?
  args.concat ["-o", script.output_path] unless script.output_path.nil?
  args.concat ["-e", script.error_path] unless script.error_path.nil?
  args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
  args.concat ["-p", script.queue_name] unless script.queue_name.nil?
  args.concat ["--priority", script.priority] unless script.priority.nil?
  args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
  args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
  args.concat ['--qos', script.qos] unless script.qos.nil?
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
  args.concat ['-n', script.cores] unless script.cores.nil?
  # ignore nodes, don't know how to do this for slurm

  # Set dependencies
  depend = []
  depend << "after:#{after.join(":")}"           unless after.empty?
  depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
  depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
  depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
  args.concat ["-d", depend.join(",")]               unless depend.empty?

  # Set environment variables
  env = script.job_environment || {}
  args.concat ["--export", export_arg(env, script.copy_environment?)]

  # Set native options
  args.concat script.native if script.native

  # Set content
  content = if script.shell_path.nil?
              script.content
            else
              "#!#{script.shell_path}\n#{script.content}"
            end

  # Submit job
  @slurm.submit_string(content, args: args, env: env)
rescue Batch::Error => e
  raise JobAdapterError, e.message
end