Class: OodCore::Job::Adapters::Slurm

Inherits:
OodCore::Job::Adapter show all
Defined in:
lib/ood_core/job/adapters/slurm.rb

Overview

An adapter object that describes the communication with a Slurm resource manager for job management.

Defined Under Namespace

Classes: Batch

Constant Summary collapse

STATE_MAP =

Mapping of state codes for Slurm

{
  'BF' => :completed,  # BOOT_FAIL
  'CA' => :completed,  # CANCELLED
  'CD' => :completed,  # COMPLETED
  'CF' => :queued,     # CONFIGURING
  'CG' => :running,    # COMPLETING
  'F'  => :completed,  # FAILED
  'NF' => :completed,  # NODE_FAIL
  'PD' => :queued,     # PENDING
  'PR' => :suspended,  # PREEMPTED
  'RV' => :completed,  # REVOKED
  'R'  => :running,    # RUNNING
  'SE' => :completed,  # SPECIAL_EXIT
  'ST' => :running,    # STOPPED
  'S'  => :suspended,  # SUSPENDED
  'TO' => :completed,  # TIMEOUT
  'OOM' => :completed, # OUT_OF_MEMORY

  'BOOT_FAIL'     => :completed,
  'CANCELED'      => :completed,
  'COMPLETED'     => :completed,
  'DEADLINE'      => :completed,
  'FAILED'        => :completed,
  'NODE_FAIL'     => :completed,
  'OUT_OF_MEMORY' => :completed,
  'PENDING'       => :queued,
  'PREEMPTED'     => :completed,
  'RUNNING'       => :running,
  'REQUEUED'      => :queued,
  'REVOKED'       => :completed,
  'SUSPENDED'     => :suspended,
  'TIMEOUT'       => :completed,
}

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from OodCore::Job::Adapter

#info_all_each, #info_where_owner_each, #job_name_illegal_chars, #sanitize_job_name, #supports_job_arrays?

Constructor Details

#initialize(opts = {}) ⇒ Slurm

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Returns a new instance of Slurm.

Parameters:

  • opts (#to_h) (defaults to: {}) —

    the options defining this adapter

Options Hash (opts):

  • :slurm (Batch) —

    The Slurm batch object

See Also:



590
591
592
593
594
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 590

def initialize(opts = {})
  o = opts.to_h.symbolize_keys

  @slurm = o.fetch(:slurm) { raise ArgumentError, "No slurm object specified. Missing argument: slurm" }
end

Class Method Details

.gpus_from_gres(gres) ⇒ Integer

Get integer representing the number of gpus used by a node or job, calculated from gres string

Returns:

  • (Integer) —

    the number of gpus in gres



49
50
51
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 49

def self.gpus_from_gres(gres)
  gres.to_s.scan(/gpu[^(,]*[:=](\d+)/).flatten.map(&:to_i).sum
end

Instance Method Details

#accounts ⇒ Array<String>

Retrieve the accounts available to use for the current user.

Returns:

  • (Array<String>) —

    the accounts available to the user.



686
687
688
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 686

def accounts
  @slurm.accounts
end

#cluster_info ⇒ Hash

Retrieve info about active and total cpus, gpus, and nodes

Returns:

  • (Hash) —

    information about cluster usage



679
680
681
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 679

def cluster_info
  @slurm.get_cluster_info
end

#delete(id) ⇒ void

This method returns an undefined value.

Delete the submitted job

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



840
841
842
843
844
845
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 840

def delete(id)
  @slurm.delete_job(id.to_s)
rescue Batch::Error => e
  # assume successful job deletion if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#directive_prefix ⇒ Object 



847
848
849
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 847

def directive_prefix
  '#SBATCH'
end

#hold(id) ⇒ void

This method returns an undefined value.

Put the submitted job on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



816
817
818
819
820
821
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 816

def hold(id)
  @slurm.hold_job(id.to_s)
rescue Batch::Error => e
  # assume successful job hold if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#info(id) ⇒ Info

Retrieve job info from the resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

  • (Info) —

    information describing submitted job

Raises:

See Also:



746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 746

def info(id)
  id = id.to_s
  info_ary = @slurm.get_jobs(id: id).map do |v|
    parse_job_info(v)
  end

  # If no job was found we assume that it has completed
  info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Info.new(
      id: id,
      status: :completed
    )
  else
    raise JobAdapterError, e.message
  end
end

#info_all(attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs from the resource manager

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:

See Also:



694
695
696
697
698
699
700
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 694

def info_all(attrs: nil)
  @slurm.get_jobs(attrs: attrs).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#info_historic(opts: {}) ⇒ Array<Info>

Retrieve historic info for all completed jobs from the resource manager.

Known options: job_ids [Array<#to_s>] optional list of job ids to filter the results. states [Array<#to_s>] optional list of job state codes. Selects jobs based on their state during the time period given. from [#to_s] optional date string to filter jobs in any state after the specified time. If states are provided, filter jobs in these states after this period to [#to_s] optional date string to filter jobs in any state before the specified time. If states are provided, filter jobs in these states before this period. show_steps [#Boolean] optional boolean to filter job steps from the results.

Returns:

  • (Array<Info>) —

    information describing submitted jobs

See Also:



716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 716

def info_historic(opts: {})
  job_ids    = opts.fetch(:job_ids, [])
  states     = opts.fetch(:states, [])
  from       = opts.fetch(:from, nil)
  to         = opts.fetch(:to, nil)
  show_steps = opts.fetch(:show_steps, false)
  @slurm.sacct_info(job_ids, states, from, to, show_steps).map do |v|
    Info.new(
      id: v[:job_id],
      status: get_state(v[:state]),
      job_name: v[:job_name],
      job_owner: v[:user],
      procs: v[:alloc_cpus],
      queue_name: v[:partition],
      wallclock_time: duration_in_seconds(v[:elapsed]),
      wallclock_limit: duration_in_seconds(v[:time_limit]),
      cpu_time: duration_in_seconds(v[:total_cpu]),
      submission_time: parse_time(v[:submit_time]),
      dispatch_time: parse_time(v[:start_time]),
      native: v,
      gpus: self.class.gpus_from_gres(v[:gres])
    )
  end
end

#info_where_owner(owner, attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs for a given owner or owners from the resource manager

Parameters:

  • owner (#to_s, Array<#to_s>) —

    the owner(s) of the jobs

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:



771
772
773
774
775
776
777
778
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 771

def info_where_owner(owner, attrs: nil)
  owner = Array.wrap(owner).map(&:to_s).join(',')
  @slurm.get_jobs(owner: owner).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#nodes ⇒ Object 



855
856
857
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 855

def nodes
  @slurm.nodes
end

#queues ⇒ Object 



851
852
853
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 851

def queues
  @slurm.queues
end

#release(id) ⇒ void

This method returns an undefined value.

Release the job that is on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



828
829
830
831
832
833
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 828

def release(id)
  @slurm.release_job(id.to_s)
rescue Batch::Error => e
  # assume successful job release if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#status(id) ⇒ Status

Retrieve job status from resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

Raises:

See Also:



785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 785

def status(id)
  id = id.to_s
  jobs = @slurm.get_jobs(
    id: id,
    attrs: [:job_id, :array_job_task_id, :state_compact]
  )
  # A job id can return multiple jobs if it corresponds to a job array
  # id, so we need to find the job that corresponds to the given job id
  # (if we can't find it, we assume it has completed)
  #
  # Match against the job id or the formatted job & task id "1234_0"
  if job = jobs.detect { |j| j[:job_id] == id || j[:array_job_task_id] == id }
    Status.new(state: get_state(job[:state_compact]))
  else
    # set completed status if can't find job id
    Status.new(state: :completed)
  end
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Status.new(state: :completed)
  else
    raise JobAdapterError, e.message
  end
end

#submit(script, after: [], afterok: [], afternotok: [], afterany: []) ⇒ String

Submit a job with the attributes defined in the job template instance

Parameters:

  • script (Script) —

    script object that describes the script and attributes for the submitted job

  • after (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution at any point after dependent jobs have started execution

  • afterok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with no errors

  • afternotok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with errors

  • afterany (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution after dependent jobs have terminated

Returns:

  • (String) —

    the job id returned after successfully submitting a job

Raises:

See Also:



611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 611

def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
  after      = Array(after).map(&:to_s)
  afterok    = Array(afterok).map(&:to_s)
  afternotok = Array(afternotok).map(&:to_s)
  afterany   = Array(afterany).map(&:to_s)

  # Set sbatch options
  args = []
  # ignore args, don't know how to do this for slurm
  args.concat ["-H"] if script.submit_as_hold
  args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
  args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
  args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
  if script.email_on_started && script.email_on_terminated
    args.concat ["--mail-type", "ALL"]
  elsif script.email_on_started
    args.concat ["--mail-type", "BEGIN"]
  elsif script.email_on_terminated
    args.concat ["--mail-type", "END"]
  elsif script.email_on_started == false && script.email_on_terminated == false
    args.concat ["--mail-type", "NONE"]
  end
  args.concat ["-J", script.job_name] unless script.job_name.nil?
  args.concat ["-i", script.input_path] unless script.input_path.nil?
  args.concat ["-o", script.output_path] unless script.output_path.nil?
  args.concat ["-e", script.error_path] unless script.error_path.nil?
  args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
  args.concat ["-p", script.queue_name] unless script.queue_name.nil?
  args.concat ["--priority", script.priority] unless script.priority.nil?
  args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
  args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
  args.concat ['--qos', script.qos] unless script.qos.nil?
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
  args.concat ['-n', script.cores] unless script.cores.nil?
  # ignore nodes, don't know how to do this for slurm

  # Set dependencies
  depend = []
  depend << "after:#{after.join(":")}"           unless after.empty?
  depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
  depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
  depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
  args.concat ["-d", depend.join(",")]               unless depend.empty?

  # Set environment variables
  env = script.job_environment || {}
  args.concat ["--export", export_arg(env, script.copy_environment?)]

  # Set native options
  args.concat script.native if script.native

  # Set content
  content = if script.shell_path.nil?
              script.content
            else
              "#!#{script.shell_path}\n#{script.content}"
            end

  # Submit job
  @slurm.submit_string(content, args: args, env: env)
rescue Batch::Error => e
  raise JobAdapterError, e.message
end