S3 inputs

This example shows how to retrieve inputs from somewhere in s3 and bind those to a job.

This example requires that you have boto3 installed in your python environment.

import boto3
from pathlib import Path
from pyro_dash_py import PyroDash
import os

apikey = os.environ.get("PYRO_API_KEY")
if apikey is None:
raise SystemError("PYRO_API_KEY missing!")

# initialize comms with the pyro-dashboard api

pyro = PyroDash(
host="https://dev-api.dashboard.pyrologix.com",
email="jack.campanella@pyrologix.com",
apikey=apikey,
)

job = pyro.jobs.create("wildest")
job.set_name("small wildest test")
print(f"created job: {job.name} ({job.id})")

# NOTE: by default boto3 will use whatever aws credentials you have

# configured at ~/.aws/config but you can also provide credentials

# to the client

client = boto3.client(
"s3", # aws_access_key_id=ACCESS_KEY, <-- Credentials here # aws_secret_access_key=SECRET_KEY, # aws_session_token=SESSION_TOKEN
)

bucket = "pyrologix-pub-data"
s3_keys = [
"pyrodashpy_test/asp.tif",
"pyrodashpy_test/cbd.tif",
"pyrodashpy_test/cbh.tif",
"pyrodashpy_test/cc.tif",
"pyrodashpy_test/ch.tif",
"pyrodashpy_test/dem.tif",
"pyrodashpy_test/fm40.tif",
"pyrodashpy_test/slp.tif",
]

for key in s3_keys: # first download the file from s3
fname = key.split("/")[1]
path = Path.cwd() / fname
print(f"downloading: {key} -> {path}")
client.download_file(bucket, key, str(path))

    # now we can add it to our job
    print(f"adding {path} to job {job.name}")
    job.add_file(str(path))

    # remove the file from our system
    path.unlink()

# optionally show our job's inputs

for input in job.list_inputs():
print(f"Input: {input.name} ({input.id})")