Class: OodCore::Job::Factory

Inherits:
Object
  • Object
show all
Defined in:
lib/ood_core/job/factory.rb,
lib/ood_core/job/adapters/ccq.rb,
lib/ood_core/job/adapters/lsf.rb,
lib/ood_core/job/adapters/sge.rb,
lib/ood_core/job/adapters/psij.rb,
lib/ood_core/job/adapters/coder.rb,
lib/ood_core/job/adapters/slurm.rb,
lib/ood_core/job/adapters/pbspro.rb,
lib/ood_core/job/adapters/torque.rb,
lib/ood_core/job/adapters/systemd.rb,
lib/ood_core/job/adapters/htcondor.rb,
lib/ood_core/job/adapters/kubernetes.rb,
lib/ood_core/job/adapters/linux_host.rb,
lib/ood_core/job/adapters/fujitsu_tcs.rb

Overview

A factory that builds job adapter objects from a configuration.

Class Method Summary collapse

Class Method Details

.build(config) ⇒ Adapter

Build a job adapter from a configuration

Parameters:

  • config (#to_h) —

    configuration describing job adapter

Options Hash (config):

  • :adapter (#to_s) —

    The job adapter to use

Returns:

  • (Adapter) —

    the job adapter object

Raises:



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/ood_core/job/factory.rb', line 16

def build(config)
  c = config.to_h.symbolize_keys

  adapter = c.fetch(:adapter) { raise AdapterNotSpecified, "job configuration does not specify adapter" }.to_s

  path_to_adapter = "ood_core/job/adapters/#{adapter}"
  begin
    require path_to_adapter
  rescue Gem::LoadError => e
    raise Gem::LoadError, "Specified '#{adapter}' for job adapter, but the gem is not loaded."
  rescue LoadError => e
    raise LoadError, "Could not load '#{adapter}'. Make sure that the job adapter in the configuration file is valid."
  end

  adapter_method = "build_#{adapter}"

  unless respond_to?(adapter_method)
    raise AdapterNotFound, "job configuration specifies nonexistent #{adapter} adapter"
  end

  send(adapter_method, c)
end

.build_ccq(config) ⇒ Object

Build the Cloudy Cluster adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :image (Object) — default: nil —

    The default VM image to use

  • :cloud (Object) — default: gcp —

    The cloud provider being used [gcp,aws]

  • :scheduler (Object) — default: nil —

    The name of the scheduler to use

  • :sge_root (Object) — default: nil —

    Path to SGE root, note that

  • :bin (#to_h) — default: nil —

    Path to CC client binaries

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to CC client executables



17
18
19
 
# File 'lib/ood_core/job/adapters/ccq.rb', line 17

def self.build_ccq(config)
  Adapters::CCQ.new(config.to_h.symbolize_keys)
end

.build_coder(config) ⇒ Object 



15
16
17
18
19
20
 
# File 'lib/ood_core/job/adapters/coder.rb', line 15

def self.build_coder(config)
  config = config.to_h.symbolize_keys
  credentials = build_credentials(config[:auth] || {})
  batch = Adapters::Coder::Batch.new(config, credentials)
  Adapters::Coder.new(batch)
end

.build_credentials(auth) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/ood_core/job/adapters/coder.rb', line 22

def self.build_credentials(auth)
  begin
    auth_type = auth.to_h.fetch('cloud', 'none')

    class_name = auth_type.split(/[_-]/).map {|s| s.gsub(/\b\w/, &:upcase) }.join

    klass = Object.const_get("#{class_name}Credentials")
    klass.new(**auth.to_h.symbolize_keys)
  rescue NameError
    raise(ArgumentError, "Unsupported credentials for cloud type: #{auth_type}")
  end
end

.build_fujitsu_tcs(config) ⇒ Object

Build the Fujitsu TCS (Technical Computing Suite) adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :bin (Object) — default: nil —

    Path to Fujitsu TCS resource manager binaries

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to Fujitsu TCS resource manager executables

  • :working_dir (Object) — default: nil —

    Working directory for submitting a batch script



16
17
18
19
20
21
22
23
 
# File 'lib/ood_core/job/adapters/fujitsu_tcs.rb', line 16

def self.build_fujitsu_tcs(config)
  c = config.to_h.symbolize_keys
  bin           = c.fetch(:bin, nil)
  bin_overrides = c.fetch(:bin_overrides, {})
  working_dir   = c.fetch(:working_dir, nil)
  fujitsu_tcs   = Adapters::Fujitsu_TCS::Batch.new(bin: bin, bin_overrides: bin_overrides, working_dir: working_dir)
  Adapters::Fujitsu_TCS.new(fujitsu_tcs: fujitsu_tcs)
end

.build_htcondor(config) ⇒ Object

Build the HTCondor adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :bin (Object) — default: nil —

    Path to HTCondor client binaries

  • :submit_host (Object) — default: "" —

    Submit job on login node via ssh

  • :strict_host_checking (Object) — default: true —

    Whether to use strict host checking when ssh to submit_host



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/ood_core/job/adapters/htcondor.rb', line 18

def self.build_htcondor(config)
    c = config.to_h.symbolize_keys
    bin                  = c.fetch(:bin, nil)
    bin_overrides        = c.fetch(:bin_overrides, {})
    submit_host          = c.fetch(:submit_host, "")
    strict_host_checking = c.fetch(:strict_host_checking, true)
    default_universe = c.fetch(:default_universe, "vanilla")
    default_docker_image = c.fetch(:default_docker_image, "ubuntu:latest")
    user_group_map = c.fetch(:user_group_map, nil)
    cluster = c.fetch(:cluster, "")
    additional_attributes = c.fetch(:additional_attributes, {})
    htcondor = Adapters::HTCondor::Batch.new(bin: bin, bin_overrides: bin_overrides,
        submit_host: submit_host, strict_host_checking: strict_host_checking,
        default_universe: default_universe,
        default_docker_image: default_docker_image,
        user_group_map: user_group_map,
        cluster: cluster,
        additional_attributes: additional_attributes,
        )
    Adapters::HTCondor.new(htcondor: htcondor)
end

.build_kubernetes(config) ⇒ Object 



9
10
11
12
 
# File 'lib/ood_core/job/adapters/kubernetes.rb', line 9

def self.build_kubernetes(config)
  batch = Adapters::Kubernetes::Batch.new(config.to_h.symbolize_keys)
  Adapters::Kubernetes.new(batch)
end

.build_linux_host(config) ⇒ Object

Build the LinuxHost adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :contain (Object) — default: false —

    Pass --contain flag to Singularity; allows overriding bind mounts in singularity.conf

  • :debug (Object) — default: false —

    Use the adapter in a debug mode

  • :max_timeout (Object) — default: nil —

    The longest 'wall_clock' permissible

  • :singularity_bin (Object) — default: '/usr/bin/singularity' —

    The path to the Singularity executable

  • :singularity_bindpath (Object) — default: '/etc,/media,/mnt,/opt,/srv,/usr,/var,/users' —

    A comma delimited list of paths to bind between the host and the guest

  • :singularity_image (Object) —

    The path to the Singularity image to use

  • :ssh_hosts (Object) — default: nil —

    The list of permissable hosts, defaults to :submit_host

  • :strict_host_checking (Object) — default: true —

    Set to false to disable strict host checking and updating the known_hosts file

  • :submit_host (Object) —

    The SSH target to connect to, may be the head of a round-robin

  • :tmux_bin (Object) — default: '/usr/bin/tmux' —

    The path to the Tmux executable



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/ood_core/job/adapters/linux_host.rb', line 23

def self.build_linux_host(config)
  c = config.to_h.symbolize_keys
  contain = c.fetch(:contain, false)
  debug = c.fetch(:debug, false)
  max_timeout = c.fetch(:max_timeout, nil)
  singularity_bin = c.fetch(:singularity_bin, '/usr/bin/singularity')
  singularity_bindpath = c.fetch(:singularity_bindpath, '/etc,/media,/mnt,/opt,/srv,/usr,/var,/users')
  singularity_image = c[:singularity_image]
  ssh_hosts = c.fetch(:ssh_hosts, [c[:submit_host]])
  strict_host_checking = c.fetch(:strict_host_checking, true)
  submit_host = c[:submit_host]
  tmux_bin = c.fetch(:tmux_bin, '/usr/bin/tmux')

  Adapters::LinuxHost.new(
    ssh_hosts: ssh_hosts,
    launcher: Adapters::LinuxHost::Launcher.new(
      contain: contain,
      debug: debug,
      max_timeout: max_timeout,
      singularity_bin: singularity_bin,
      singularity_bindpath: singularity_bindpath,  # '/etc,/media,/mnt,/opt,/srv,/usr,/var,/users',
      singularity_image: singularity_image,
      ssh_hosts: ssh_hosts,
      strict_host_checking: strict_host_checking,
      submit_host: submit_host,
      tmux_bin: tmux_bin,
    )
  )
end

.build_lsf(config) ⇒ Object

Build the Lsf adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :bindir (#to_s) — default: '' —

    Path to lsf client bin dir

  • :libdir (#to_s) — default: '' —

    Path to lsf client lib dir

  • :envdir (#to_s) — default: '' —

    Path to lsf client conf dir

  • :serverdir (#to_s) — default: '' —

    Path to lsf client etc dir

  • :cluster (#to_s) — default: '' —

    name of cluster, if in multi-cluster mode

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to LSF client executables

  • :submit_host (#to_s) — default: '' —

    Host to submit commands to



18
19
20
21
 
# File 'lib/ood_core/job/adapters/lsf.rb', line 18

def self.build_lsf(config)
  batch = Adapters::Lsf::Batch.new(**config.to_h.symbolize_keys)
  Adapters::Lsf.new(batch: batch)
end

.build_pbspro(config) ⇒ Object

Build the PBS Pro adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :host (Object) — default: nil —

    The batch server host

  • :submit_host (Object) — default: "" —

    The login node where the job is submitted

  • :strict_host_checking (Object) — default: true —

    Whether to use strict host checking when ssh to submit_host

  • :exec (Object) — default: nil —

    Path to PBS Pro executables

  • :qstat_factor (Object) — default: nil —

    Deciding factor on how to call qstat for a user

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to PBS Pro client executables



21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/ood_core/job/adapters/pbspro.rb', line 21

def self.build_pbspro(config)
  c = config.to_h.compact.symbolize_keys
  host                 = c.fetch(:host, nil)
  submit_host          = c.fetch(:submit_host, "")
  strict_host_checking = c.fetch(:strict_host_checking, true)
  pbs_exec             = c.fetch(:exec, nil)
  qstat_factor         = c.fetch(:qstat_factor, nil)
  bin_overrides         = c.fetch(:bin_overrides, {})
  pbspro = Adapters::PBSPro::Batch.new(host: host, submit_host: submit_host, strict_host_checking: strict_host_checking, pbs_exec: pbs_exec, bin_overrides: bin_overrides)
  Adapters::PBSPro.new(pbspro: pbspro, qstat_factor: qstat_factor)
end

.build_psij(config) ⇒ Object

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :bin (Object) — default: nil —

    Path to PSIJ binaries

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to PSIJ executables



19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/ood_core/job/adapters/psij.rb', line 19

def self.build_psij(config)
  c = config.to_h.symbolize_keys
  cluster              = c.fetch(:cluster, nil)
  conf                 = c.fetch(:conf, nil)
  bin                  = c.fetch(:bin, nil)
  bin_overrides        = c.fetch(:bin_overrides, {})
  submit_host          = c.fetch(:submit_host, "")
  strict_host_checking = c.fetch(:strict_host_checking, true)
  executor             = c.fetch(:executor, nil)
  queue_name           = c.fetch(:queue_name, nil)
  psij = Adapters::PSIJ::Batch.new(cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides, submit_host: submit_host, strict_host_checking: strict_host_checking, executor: executor, queue_name: queue_name)
  Adapters::PSIJ.new(psij: psij)
end

.build_sge(config) ⇒ Object

Build the Sun Grid Engine adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :cluster (Object) — default: nil —

    The cluster to communicate with

  • :conf (Object) — default: nil —

    Path to the SGE conf

  • :bin (Object) — default: nil —

    Path to SGE client binaries

  • :sge_root (Object) — default: nil —

    Path to SGE root, note that

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to SGE client executables this may be nil, but must be set to use the DRMAA API, and there is a severe performance penalty calling Sge#info without using DRMAA.



18
19
20
21
 
# File 'lib/ood_core/job/adapters/sge.rb', line 18

def self.build_sge(config)
  batch = Adapters::Sge::Batch.new(config.to_h.symbolize_keys)
  Adapters::Sge.new(batch: batch)
end

.build_slurm(config) ⇒ Object

Build the Slurm adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :cluster (Object) — default: nil —

    The cluster to communicate with

  • :conf (Object) — default: nil —

    Path to the slurm conf

  • :bin (Object) — default: nil —

    Path to slurm client binaries

  • :bin_overrides (#to_h) — default: {} —

    Optional overrides to Slurm client executables

  • :submit_host (Object) — default: "" —

    Submit job on login node via ssh

  • :strict_host_checking (Object) — default: true —

    Whether to use strict host checking when ssh to submit_host



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 20

def self.build_slurm(config)
  c = config.to_h.symbolize_keys
  cluster              = c.fetch(:cluster, nil)
  conf                 = c.fetch(:conf, nil)
  bin                  = c.fetch(:bin, nil)
  bin_overrides        = c.fetch(:bin_overrides, {})
  submit_host          = c.fetch(:submit_host, "")
  strict_host_checking = c.fetch(:strict_host_checking, true)
  id                   = c.fetch(:id, 'unknown')

  slurm = Adapters::Slurm::Batch.new(
    cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides,
    submit_host: submit_host, strict_host_checking: strict_host_checking,
    id: id
  )
  Adapters::Slurm.new(slurm: slurm)
end

.build_systemd(config) ⇒ Object

Build the LinuxSystemd adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :debug (Object) — default: false —

    Use the adapter in a debug mode

  • :site_timeout (Object) — default: nil —

    The longest 'wall_clock' permissible

  • :max_timeout (Object) — default: nil —

    Deprecated alias for :site_timeout

  • :ssh_hosts (Object) — default: nil —

    The list of permissable hosts, defaults to :submit_host

  • :strict_host_checking (Object) — default: true —

    Set to false to disable strict host checking and updating the known_hosts file

  • :submit_host (Object) —

    The SSH target to connect to, may be the head of a round-robin

  • :ssh_keyfile (Object) —

    The SSH Key file to use as identity.



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/ood_core/job/adapters/systemd.rb', line 20

def self.build_systemd(config)
  c = config.to_h.symbolize_keys
  debug = c.fetch(:debug, false)
  site_timeout = c.fetch(:site_timeout, c.fetch(:max_timeout, nil))
  ssh_hosts = c.fetch(:ssh_hosts, [c[:submit_host]])
  strict_host_checking = c.fetch(:strict_host_checking, true)
  submit_host = c[:submit_host]
  ssh_keyfile = c.fetch(:ssh_keyfile, "")

  Adapters::LinuxSystemd.new(
    ssh_hosts: ssh_hosts,
    launcher: Adapters::LinuxSystemd::Launcher.new(
      debug: debug,
      site_timeout: site_timeout,
      ssh_hosts: ssh_hosts,
      strict_host_checking: strict_host_checking,
      submit_host: submit_host,
      ssh_keyfile: ssh_keyfile,
    )
  )
end

.build_torque(config) ⇒ Object

Build the Torque adapter from a configuration

Parameters:

  • config (#to_h) —

    the configuration for job adapter

Options Hash (config):

  • :host (#to_s) —

    The batch server host

  • :submit_host (#to_s) —

    The login node to submit the job via ssh

  • :lib (#to_s) — default: '' —

    Path to torque client libraries

  • :bin (#to_s) — default: '' —

    Path to torque client binaries

  • :custom_bin (#to_h) — default: {} —

    Optional overrides to Torque client executables



17
18
19
20
21
22
23
24
25
26
 
# File 'lib/ood_core/job/adapters/torque.rb', line 17

def self.build_torque(config)
  c = config.to_h.symbolize_keys
  host = c.fetch(:host) { raise ArgumentError, "No host specified. Missing argument: host" }.to_s
  submit_host = c.fetch(:submit_host, "").to_s
  lib  = c.fetch(:lib, "").to_s
  bin  = c.fetch(:bin, "").to_s
  custom_bin = c.fetch(:custom_bin, {})
  pbs  = Adapters::Torque::Batch.new(host: host, submit_host: submit_host, lib: lib, bin: bin, custom_bin: custom_bin)
  Adapters::Torque.new(pbs: pbs)
end