Class: OodCore::Job::Adapters::Slurm

Inherits:
OodCore::Job::Adapter show all
Defined in:
lib/ood_core/job/adapters/slurm.rb

Overview

An adapter object that describes the communication with a Slurm resource manager for job management.

Defined Under Namespace

Classes: Batch

Constant Summary collapse

STATE_MAP =

Mapping of state codes for Slurm

{
  'BF' => :completed,  # BOOT_FAIL
  'CA' => :completed,  # CANCELLED
  'CD' => :completed,  # COMPLETED
  'CF' => :queued,     # CONFIGURING
  'CG' => :running,    # COMPLETING
  'F'  => :completed,  # FAILED
  'NF' => :completed,  # NODE_FAIL
  'PD' => :queued,     # PENDING
  'PR' => :suspended,  # PREEMPTED
  'RV' => :completed,  # REVOKED
  'R'  => :running,    # RUNNING
  'SE' => :completed,  # SPECIAL_EXIT
  'ST' => :running,    # STOPPED
  'S'  => :suspended,  # SUSPENDED
  'TO' => :completed,  # TIMEOUT
  'OOM' => :completed, # OUT_OF_MEMORY

  'BOOT_FAIL'     => :completed,
  'CANCELLED'     => :completed,
  'COMPLETED'     => :completed,
  'DEADLINE'      => :completed,
  'FAILED'        => :completed,
  'NODE_FAIL'     => :completed,
  'OUT_OF_MEMORY' => :completed,
  'PENDING'       => :queued,
  'PREEMPTED'     => :completed,
  'RUNNING'       => :running,
  'REQUEUED'      => :queued,
  'REVOKED'       => :completed,
  'SUSPENDED'     => :suspended,
  'TIMEOUT'       => :completed,
}

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from OodCore::Job::Adapter

#info_all_each, #info_where_owner_each, #job_name_illegal_chars, #sanitize_job_name, #supports_job_arrays?

Constructor Details

#initialize(opts = {}) ⇒ Slurm

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Returns a new instance of Slurm.

Parameters:

  • opts (#to_h) (defaults to: {}) —

    the options defining this adapter

Options Hash (opts):

  • :slurm (Batch) —

    The Slurm batch object

See Also:



598
599
600
601
602
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 598

def initialize(opts = {})
  o = opts.to_h.symbolize_keys

  @slurm = o.fetch(:slurm) { raise ArgumentError, "No slurm object specified. Missing argument: slurm" }
end

Class Method Details

.gpus_from_gres(gres) ⇒ Integer

Get integer representing the number of gpus used by a node or job, calculated from gres string

Returns:

  • (Integer) —

    the number of gpus in gres



49
50
51
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 49

def self.gpus_from_gres(gres)
  gres.to_s.scan(/gpu[s:]*[\w()-]*[=:]?(\d+)(?:[(,]|$)/).flatten.map(&:to_i).sum
end

Instance Method Details

#accounts ⇒ Array<String>

Retrieve the accounts available to use for the current user.

Returns:

  • (Array<String>) —

    the accounts available to the user.



694
695
696
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 694

def accounts
  @slurm.accounts
end

#cluster_info ⇒ Hash

Retrieve info about active and total cpus, gpus, and nodes

Returns:

  • (Hash) —

    information about cluster usage



687
688
689
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 687

def cluster_info
  @slurm.get_cluster_info
end

#delete(id) ⇒ void

This method returns an undefined value.

Delete the submitted job

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



848
849
850
851
852
853
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 848

def delete(id)
  @slurm.delete_job(id.to_s)
rescue Batch::Error => e
  # assume successful job deletion if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#directive_prefix ⇒ Object 



855
856
857
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 855

def directive_prefix
  '#SBATCH'
end

#hold(id) ⇒ void

This method returns an undefined value.

Put the submitted job on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



824
825
826
827
828
829
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 824

def hold(id)
  @slurm.hold_job(id.to_s)
rescue Batch::Error => e
  # assume successful job hold if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#info(id) ⇒ Info

Retrieve job info from the resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

  • (Info) —

    information describing submitted job

Raises:

See Also:



754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 754

def info(id)
  id = id.to_s
  info_ary = @slurm.get_jobs(id: id).map do |v|
    parse_job_info(v)
  end

  # If no job was found we assume that it has completed
  info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Info.new(
      id: id,
      status: :completed
    )
  else
    raise JobAdapterError, e.message
  end
end

#info_all(attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs from the resource manager

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:

See Also:



702
703
704
705
706
707
708
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 702

def info_all(attrs: nil)
  @slurm.get_jobs(attrs: attrs).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#info_historic(opts: {}) ⇒ Array<Info>

Retrieve historic info for all completed jobs from the resource manager.

Known options: job_ids [Array<#to_s>] optional list of job ids to filter the results. states [Array<#to_s>] optional list of job state codes. Selects jobs based on their state during the time period given. from [#to_s] optional date string to filter jobs in any state after the specified time. If states are provided, filter jobs in these states after this period to [#to_s] optional date string to filter jobs in any state before the specified time. If states are provided, filter jobs in these states before this period. show_steps [#Boolean] optional boolean to filter job steps from the results.

Returns:

  • (Array<Info>) —

    information describing submitted jobs

See Also:



724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 724

def info_historic(opts: {})
  job_ids    = opts.fetch(:job_ids, [])
  states     = opts.fetch(:states, [])
  from       = opts.fetch(:from, nil)
  to         = opts.fetch(:to, nil)
  show_steps = opts.fetch(:show_steps, false)
  @slurm.sacct_info(job_ids, states, from, to, show_steps).map do |v|
    Info.new(
      id: v[:job_id],
      status: get_state(v[:state]),
      job_name: v[:job_name],
      job_owner: v[:user],
      procs: v[:alloc_cpus],
      queue_name: v[:partition],
      wallclock_time: duration_in_seconds(v[:elapsed]),
      wallclock_limit: duration_in_seconds(v[:time_limit]),
      cpu_time: duration_in_seconds(v[:total_cpu]),
      submission_time: parse_time(v[:submit_time]),
      dispatch_time: parse_time(v[:start_time]),
      native: v,
      gpus: self.class.gpus_from_gres(v[:gres])
    )
  end
end

#info_where_owner(owner, attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs for a given owner or owners from the resource manager

Parameters:

  • owner (#to_s, Array<#to_s>) —

    the owner(s) of the jobs

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:



779
780
781
782
783
784
785
786
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 779

def info_where_owner(owner, attrs: nil)
  owner = Array.wrap(owner).map(&:to_s).join(',')
  @slurm.get_jobs(owner: owner).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#nodes ⇒ Object 



863
864
865
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 863

def nodes
  @slurm.nodes
end

#queues ⇒ Object 



859
860
861
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 859

def queues
  @slurm.queues
end

#release(id) ⇒ void

This method returns an undefined value.

Release the job that is on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



836
837
838
839
840
841
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 836

def release(id)
  @slurm.release_job(id.to_s)
rescue Batch::Error => e
  # assume successful job release if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#status(id) ⇒ Status

Retrieve job status from resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

Raises:

See Also:



793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 793

def status(id)
  id = id.to_s
  jobs = @slurm.get_jobs(
    id: id,
    attrs: [:job_id, :array_job_task_id, :state_compact]
  )
  # A job id can return multiple jobs if it corresponds to a job array
  # id, so we need to find the job that corresponds to the given job id
  # (if we can't find it, we assume it has completed)
  #
  # Match against the job id or the formatted job & task id "1234_0"
  if job = jobs.detect { |j| j[:job_id] == id || j[:array_job_task_id] == id }
    Status.new(state: get_state(job[:state_compact]))
  else
    # set completed status if can't find job id
    Status.new(state: :completed)
  end
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Status.new(state: :completed)
  else
    raise JobAdapterError, e.message
  end
end

#submit(script, after: [], afterok: [], afternotok: [], afterany: []) ⇒ String

Submit a job with the attributes defined in the job template instance

Parameters:

  • script (Script) —

    script object that describes the script and attributes for the submitted job

  • after (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution at any point after dependent jobs have started execution

  • afterok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with no errors

  • afternotok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with errors

  • afterany (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution after dependent jobs have terminated

Returns:

  • (String) —

    the job id returned after successfully submitting a job

Raises:

See Also:



619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 619

def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
  after      = Array(after).map(&:to_s)
  afterok    = Array(afterok).map(&:to_s)
  afternotok = Array(afternotok).map(&:to_s)
  afterany   = Array(afterany).map(&:to_s)

  # Set sbatch options
  args = []
  # ignore args, don't know how to do this for slurm
  args.concat ["-H"] if script.submit_as_hold
  args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
  args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
  args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
  if script.email_on_started && script.email_on_terminated
    args.concat ["--mail-type", "ALL"]
  elsif script.email_on_started
    args.concat ["--mail-type", "BEGIN"]
  elsif script.email_on_terminated
    args.concat ["--mail-type", "END"]
  elsif script.email_on_started == false && script.email_on_terminated == false
    args.concat ["--mail-type", "NONE"]
  end
  args.concat ["-J", script.job_name] unless script.job_name.nil?
  args.concat ["-i", script.input_path] unless script.input_path.nil?
  args.concat ["-o", script.output_path] unless script.output_path.nil?
  args.concat ["-e", script.error_path] unless script.error_path.nil?
  args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
  args.concat ["-p", script.queue_name] unless script.queue_name.nil?
  args.concat ["--priority", script.priority] unless script.priority.nil?
  args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
  args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
  args.concat ['--qos', script.qos] unless script.qos.nil?
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
  args.concat ['-n', script.cores] unless script.cores.nil?
  # ignore nodes, don't know how to do this for slurm

  # Set dependencies
  depend = []
  depend << "after:#{after.join(":")}"           unless after.empty?
  depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
  depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
  depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
  args.concat ["-d", depend.join(",")]               unless depend.empty?

  # Set environment variables
  env = script.job_environment || {}
  args.concat ["--export", export_arg(env, script.copy_environment?)]

  # Set native options
  args.concat script.native if script.native

  # Set content
  content = if script.shell_path.nil?
              script.content
            else
              "#!#{script.shell_path}\n#{script.content}"
            end

  # Submit job
  @slurm.submit_string(content, args: args, env: env)
rescue Batch::Error => e
  raise JobAdapterError, e.message
end