flux submit

Basic job submission

# Submit a sleep job
flux submit sleep 60

# Submit a job and watch the output
flux submit --watch ./interesting-job.sh

# Submit a job to run on 2 nodes, 4 tasks, and 2 cores per task
flux submit --nodes=2 --ntasks=4 --cores-per-task=2 ./script.sh

# Submit a named job to queue jellybelly
flux submit --queue=jellybelly --name=buttered-popcorn ./script.sh

# Submit with a dry run to see the jobspec
flux submit --dry-run ./highly-suspicious-job.sh

Advanced job submission

# Use flux carbon copy to submit identical jobs with different inputs
flux submit --cc="1-10" echo "Hello I am job {cc}"

# Submits scripts myscript1.sh through myscript10.sh
flux submit --cc=1-10 myscript{cc}.sh

# Bypass the key value store and write output to file with jobid
flux submit --output=job-{{id}}.out ./my_super_cool_job.py

# Make a job waitable
flux submit --flags waitable ./wait-for-me.sh

# Wait for all jobs
flux job wait --all

flux run

Basic interactive jobs

# Run a job to print the hostname
flux run hostname

# Run an interactive sleep job
flux run bash -c "echo start; sleep 5; echo done"

# Run an MPI task with options for MPI
flux run -N 1 -n 4 -c 10 -g 1 -o mpi=spectrum -o cpu-affinity=per-task -o gpu-affinity=per-task my_mpi_binary

# Remove the entire environment except for PATH
flux run --env-remove=* --env=PATH ./top-secret.sh

flux jobs

Basic job listing tool

# See all my flux jobs
flux jobs -a

# Get a return code for a specific job
flux jobs --no-header -o '{returncode}' ƒj4CgebBV

# Add another field, status
flux jobs --no-header -o '{status}:{returncode}' ƒj4CgebBV

# Show me the job ids in a particular state
flux jobs --filter=pending,running,inactive -o "{id}"

flux job

Basic interaction with jobs

# Get the ID of the last submitted job
flux job last

# Get the output of the last submitted job
flux job attach $(flux job last)

# Send a signal to a specific job
flux job kill --signal=SIGUSR1 ƒj4CgebBV

flux start

Basic start of a Flux instance

# Start a test instance of size 4
flux start --test-size=4

flux proxy

Outside a flux instance

# Connect to a running flux instance
flux proxy local:///run/flux/local bash

# Issue a command to a running flux instance
flux proxy local:///run/flux/local ./script.sh

Inside a flux instance

# Get the active FLUX_URI (socket)
echo $FLUX_URI

flux resources

Inspect the resources available to you

# View the current instance nodes and statuses
flux resource list

# View the status of the flux overlay network
flux overlay status

# How long has my flux resource been up?
flux uptime

flux batch

Run batch scripts

# Submit a batch script wrapper in /bin/sh
flux batch --job-name=test-name --wrap ./batch.sh

# Example of wrapped batch script
#!/bin/sh
# flux: -N4 -n16
flux run -n16 hostname


# Example of batch script with submission directives
#flux: -N4
#flux: --job-name=name
flux run -N4 hostname

flux cancel

Basic job cancelling

# Cancel a job based on ID
flux cancel 123456

# Cancel all jobs
flux cancelall -f

# Cancel jobs for user meatball
flux cancel --user meatball

# Cancel jobs in a state (e.g., running)
flux cancel --states=RUN

flux alloc

Create basic allocations

# Create an interactive allocation with two nodes
flux alloc -N 2 --queue=foo

# Create an allocation in the background with two nodes on queue 'foo'
flux alloc --bg -N 2 --queue=foo

flux bulksubmit

Format String Examples

# Pipe inputs into flux bulksubmit
seq 1 4 | flux bulksubmit echo {}

# Map the same inputs on the command line using GNU parallel-style ::: separator
flux bulksubmit echo {} ::: 1 2 3 4

# Remove the extension of the input string (e.g., one)
flux bulksubmit echo {.%} ::: one.txt two.txt three.txt

# Get the basename of the input string (e.g., path.txt)
flux bulksubmit echo {./} ::: /long/file/path.txt /long/file/filename.txt

# Combine the two - get the basename with extension removed (e.g., path)
flux bulksubmit echo {./%} ::: /long/file/path.txt /long/file/filename.txt

# Get the directory name of the input string (e.g., /long/file)
flux bulksubmit echo {.//} ::: /long/file/path.txt /long/file/filename.txt

# Get the input sequence number, start counting at 0
flux bulksubmit echo {seq} ::: 1 2 3 4

# Get the input sequence number, start counting at 1 (you monster)
flux bulksubmit echo {seq1} ::: 1 2 3 4

# Define a custom counter called p2
flux bulksubmit --dry-run --define=p2='2**int(x)' -n {.p2} hostname ::: $(seq 0 4)

Simple Examples

# Compress every file under ./mydir in parallel, one process per core
find ./mydir | flux bulksubmit --wait xz {}

# Do a dry run and provide multiple inputs
flux bulksubmit --dry-run /bin/bash -c 'mkdir -p ./mydir/{0}/{1}; echo created mydir/{0}/{1}' ::: a b c d ::: {1..4}

# Use carbon copy to submit identical jobs with different inputs
flux bulksubmit --dry-run --cc={0} echo {1} ::: a b c ::: 0-1 0-3 0-7

# Similar functionality with using the seq command over a different range
flux bulksubmit echo 'Hello I am job {}' ::: $(seq 1 10)

flux advanced

Debugging

# Look at flux debug messages
flux dmesg

# Look at flux debug messages in a human readable format
flux dmesg --human

Queue and Archive

# Stop the queue, wait for running jobs to finish, and dump an archive.
flux queue stop
flux queue idle
flux dump ./archive.tar.gz

Moving Files

# Example batch script to use filemap to stage and clean up a file
flux filemap map -C /data mpi.sif
# This skips rank 0, since the file is already there
flux exec -x 0 -r all flux filemap get -C /data
flux submit -n 4 --output /tmp/fluxout/job.out --error /tmp/fluxout/job.out --flags waitable singularity exec /data/mpi.sif /opt/mpitest
# This is important to have so we wait for jobs to finish!
flux queue idle
flux filemap unmap