Class: OodCore::Job::Adapters::Slurm

Inherits:
OodCore::Job::Adapter show all
Defined in:
lib/ood_core/job/adapters/slurm.rb

Overview

An adapter object that describes the communication with a Slurm resource manager for job management.

Defined Under Namespace

Classes: Batch

Constant Summary collapse

STATE_MAP =

Mapping of state codes for Slurm

{
  'BF' => :completed,  # BOOT_FAIL
  'CA' => :completed,  # CANCELLED
  'CD' => :completed,  # COMPLETED
  'CF' => :queued,     # CONFIGURING
  'CG' => :running,    # COMPLETING
  'F'  => :completed,  # FAILED
  'NF' => :completed,  # NODE_FAIL
  'PD' => :queued,     # PENDING
  'PR' => :suspended,  # PREEMPTED
  'RV' => :completed,  # REVOKED
  'R'  => :running,    # RUNNING
  'SE' => :completed,  # SPECIAL_EXIT
  'ST' => :running,    # STOPPED
  'S'  => :suspended,  # SUSPENDED
  'TO' => :completed,  # TIMEOUT
  'OOM' => :completed, # OUT_OF_MEMORY

  'BOOT_FAIL'     => :completed,
  'CANCELED'      => :completed,
  'COMPLETED'     => :completed,
  'DEADLINE'      => :completed,
  'FAILED'        => :completed,
  'NODE_FAIL'     => :completed,
  'OUT_OF_MEMORY' => :completed,
  'PENDING'       => :queued,
  'PREEMPTED'     => :completed,
  'RUNNING'       => :running,
  'REQUEUED'      => :queued,
  'REVOKED'       => :completed,
  'SUSPENDED'     => :suspended,
  'TIMEOUT'       => :completed,
}

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from OodCore::Job::Adapter

#info_all_each, #info_where_owner_each, #job_name_illegal_chars, #sanitize_job_name, #supports_job_arrays?

Constructor Details

#initialize(opts = {}) ⇒ Slurm

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Returns a new instance of Slurm.

Parameters:

  • opts (#to_h) (defaults to: {}) —

    the options defining this adapter

Options Hash (opts):

  • :slurm (Batch) —

    The Slurm batch object

See Also:



565
566
567
568
569
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 565

def initialize(opts = {})
  o = opts.to_h.symbolize_keys

  @slurm = o.fetch(:slurm) { raise ArgumentError, "No slurm object specified. Missing argument: slurm" }
end

Class Method Details

.gpus_from_gres(gres) ⇒ Integer

Get integer representing the number of gpus used by a node or job, calculated from gres string

Returns:

  • (Integer) —

    the number of gpus in gres



43
44
45
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 43

def self.gpus_from_gres(gres)
  gres.to_s.scan(/gpu[^(,]*[:=](\d+)/).flatten.map(&:to_i).sum
end

Instance Method Details

#accounts ⇒ Array<String>

Retrieve the accounts available to use for the current user.

Returns:

  • (Array<String>) —

    the accounts available to the user.



661
662
663
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 661

def accounts
  @slurm.accounts
end

#cluster_info ⇒ Hash

Retrieve info about active and total cpus, gpus, and nodes

Returns:

  • (Hash) —

    information about cluster usage



654
655
656
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 654

def cluster_info
  @slurm.get_cluster_info
end

#delete(id) ⇒ void

This method returns an undefined value.

Delete the submitted job

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



815
816
817
818
819
820
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 815

def delete(id)
  @slurm.delete_job(id.to_s)
rescue Batch::Error => e
  # assume successful job deletion if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#directive_prefix ⇒ Object 



822
823
824
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 822

def directive_prefix
  '#SBATCH'
end

#hold(id) ⇒ void

This method returns an undefined value.

Put the submitted job on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



791
792
793
794
795
796
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 791

def hold(id)
  @slurm.hold_job(id.to_s)
rescue Batch::Error => e
  # assume successful job hold if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#info(id) ⇒ Info

Retrieve job info from the resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

  • (Info) —

    information describing submitted job

Raises:

See Also:



721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 721

def info(id)
  id = id.to_s
  info_ary = @slurm.get_jobs(id: id).map do |v|
    parse_job_info(v)
  end

  # If no job was found we assume that it has completed
  info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Info.new(
      id: id,
      status: :completed
    )
  else
    raise JobAdapterError, e.message
  end
end

#info_all(attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs from the resource manager

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:

See Also:



669
670
671
672
673
674
675
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 669

def info_all(attrs: nil)
  @slurm.get_jobs(attrs: attrs).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#info_historic(opts: {}) ⇒ Array<Info>

Retrieve historic info for all completed jobs from the resource manager.

Known options: job_ids [Array<#to_s>] optional list of job ids to filter the results. states [Array<#to_s>] optional list of job state codes. Selects jobs based on their state during the time period given. from [#to_s] optional date string to filter jobs in any state after the specified time. If states are provided, filter jobs in these states after this period to [#to_s] optional date string to filter jobs in any state before the specified time. If states are provided, filter jobs in these states before this period. show_steps [#Boolean] optional boolean to filter job steps from the results.

Returns:

  • (Array<Info>) —

    information describing submitted jobs

See Also:



691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 691

def info_historic(opts: {})
  job_ids    = opts.fetch(:job_ids, [])
  states     = opts.fetch(:states, [])
  from       = opts.fetch(:from, nil)
  to         = opts.fetch(:to, nil)
  show_steps = opts.fetch(:show_steps, false)
  @slurm.sacct_info(job_ids, states, from, to, show_steps).map do |v|
    Info.new(
      id: v[:job_id],
      status: get_state(v[:state]),
      job_name: v[:job_name],
      job_owner: v[:user],
      procs: v[:alloc_cpus],
      queue_name: v[:partition],
      wallclock_time: duration_in_seconds(v[:elapsed]),
      wallclock_limit: duration_in_seconds(v[:time_limit]),
      cpu_time: duration_in_seconds(v[:total_cpu]),
      submission_time: parse_time(v[:submit_time]),
      dispatch_time: parse_time(v[:start_time]),
      native: v,
      gpus: self.class.gpus_from_gres(v[:gres])
    )
  end
end

#info_where_owner(owner, attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs for a given owner or owners from the resource manager

Parameters:

  • owner (#to_s, Array<#to_s>) —

    the owner(s) of the jobs

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:



746
747
748
749
750
751
752
753
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 746

def info_where_owner(owner, attrs: nil)
  owner = Array.wrap(owner).map(&:to_s).join(',')
  @slurm.get_jobs(owner: owner).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#nodes ⇒ Object 



830
831
832
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 830

def nodes
  @slurm.nodes
end

#queues ⇒ Object 



826
827
828
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 826

def queues
  @slurm.queues
end

#release(id) ⇒ void

This method returns an undefined value.

Release the job that is on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



803
804
805
806
807
808
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 803

def release(id)
  @slurm.release_job(id.to_s)
rescue Batch::Error => e
  # assume successful job release if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#status(id) ⇒ Status

Retrieve job status from resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

Raises:

See Also:



760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 760

def status(id)
  id = id.to_s
  jobs = @slurm.get_jobs(
    id: id,
    attrs: [:job_id, :array_job_task_id, :state_compact]
  )
  # A job id can return multiple jobs if it corresponds to a job array
  # id, so we need to find the job that corresponds to the given job id
  # (if we can't find it, we assume it has completed)
  #
  # Match against the job id or the formatted job & task id "1234_0"
  if job = jobs.detect { |j| j[:job_id] == id || j[:array_job_task_id] == id }
    Status.new(state: get_state(job[:state_compact]))
  else
    # set completed status if can't find job id
    Status.new(state: :completed)
  end
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Status.new(state: :completed)
  else
    raise JobAdapterError, e.message
  end
end

#submit(script, after: [], afterok: [], afternotok: [], afterany: []) ⇒ String

Submit a job with the attributes defined in the job template instance

Parameters:

  • script (Script) —

    script object that describes the script and attributes for the submitted job

  • after (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution at any point after dependent jobs have started execution

  • afterok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with no errors

  • afternotok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with errors

  • afterany (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution after dependent jobs have terminated

Returns:

  • (String) —

    the job id returned after successfully submitting a job

Raises:

See Also:



586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 586

def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
  after      = Array(after).map(&:to_s)
  afterok    = Array(afterok).map(&:to_s)
  afternotok = Array(afternotok).map(&:to_s)
  afterany   = Array(afterany).map(&:to_s)

  # Set sbatch options
  args = []
  # ignore args, don't know how to do this for slurm
  args.concat ["-H"] if script.submit_as_hold
  args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
  args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
  args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
  if script.email_on_started && script.email_on_terminated
    args.concat ["--mail-type", "ALL"]
  elsif script.email_on_started
    args.concat ["--mail-type", "BEGIN"]
  elsif script.email_on_terminated
    args.concat ["--mail-type", "END"]
  elsif script.email_on_started == false && script.email_on_terminated == false
    args.concat ["--mail-type", "NONE"]
  end
  args.concat ["-J", script.job_name] unless script.job_name.nil?
  args.concat ["-i", script.input_path] unless script.input_path.nil?
  args.concat ["-o", script.output_path] unless script.output_path.nil?
  args.concat ["-e", script.error_path] unless script.error_path.nil?
  args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
  args.concat ["-p", script.queue_name] unless script.queue_name.nil?
  args.concat ["--priority", script.priority] unless script.priority.nil?
  args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
  args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
  args.concat ['--qos', script.qos] unless script.qos.nil?
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
  args.concat ['-n', script.cores] unless script.cores.nil?
  # ignore nodes, don't know how to do this for slurm

  # Set dependencies
  depend = []
  depend << "after:#{after.join(":")}"           unless after.empty?
  depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
  depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
  depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
  args.concat ["-d", depend.join(",")]               unless depend.empty?

  # Set environment variables
  env = script.job_environment || {}
  args.concat ["--export", export_arg(env, script.copy_environment?)]

  # Set native options
  args.concat script.native if script.native

  # Set content
  content = if script.shell_path.nil?
              script.content
            else
              "#!#{script.shell_path}\n#{script.content}"
            end

  # Submit job
  @slurm.submit_string(content, args: args, env: env)
rescue Batch::Error => e
  raise JobAdapterError, e.message
end