csharp Azure Batch

Posted 2021-05-11

App / Service uploads input data files to Azure storage and then creates a pool of Batch compute nodes (virtual machines).

It then creates a job that runs tasks to process each input file on the pool.

![arch](https://docs.microsoft.com/en-us/azure/batch/media/batch-technical-overview/tech_overview_03.png)

# Create resource group
az group create --name batch --location northeurope

# Create storage account
az storage account create \
    --resource-group batch \
    --name batchricstorageaccount \
    --location northeurope \
    --sku Standard_LRS

# Create batch account
az batch account create \
    --name ricbatchaccount \
    --storage-account batchricstorageaccount \
    --resource-group batch \
    --location northeurope

# To create and manage compute pools and jobs, you need to authenticate with Batch.
# Log in to the account with the az batch account login command.
# After you log in, your az batch commands use this account context.
az batch account login \
    --name ricbatchaccount \
    --resource-group batch \
    --shared-key-auth

# Create a pool of Linux compute nodes
az batch pool create \
    --id mypool \
    --vm-size Standard_A1_v2 \
    --target-dedicated-nodes 2 \
    --image canonical:ubuntuserver:16.04-LTS \
    --node-agent-sku-id "batch.node.ubuntu 16.04"

# To see the status of the pool
az batch pool show --pool-id mypool \
    --query "allocationState"

# Create a job
az batch job create \
    --id myjob \
    --pool-id mypool

# Create tasks
for i in {1..4}
do
   az batch task create \
    --task-id mytask$i \
    --job-id myjob \
    --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'"
done

# View task #1 status
az batch task show \
    --job-id myjob \
    --task-id mytask1

# View task #1 output
# To list the files created by a task on a compute node
az batch task file list \
    --job-id myjob \
    --task-id mytask1 \
    --output table

# To download one of the output files
az batch task file download \
    --job-id myjob \
    --task-id mytask1 \
    --file-path stdout.txt \
    --destination ./stdout.txt

# Clean up resources
# There is no charge for the Batch account itself.
# You are charged for pools while the nodes are running, even if no jobs are scheduled
# When you no longer need a pool, delete it (all task output on the nodes is deleted).
az batch pool delete --pool-id mypool

// From https://docs.microsoft.com/en-us/azure/batch/quick-run-dotnet

// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

// Add files to process
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blobData = container.GetBlockBlobReference(blobName);
blobData.UploadFromFileAsync(filePath).Wait();

// Create a BatchClient object to create and manage pools, jobs, and tasks in the Batch service
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
  // ...
}

// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = new ImageReference(
    publisher: "MicrosoftWindowsServer",
    offer: "WindowsServer",
    sku: "2016-datacenter-smalldisk",
    version: "latest"
);

VirtualMachineConfiguration vmConfiguration = new VirtualMachineConfiguration(
    imageReference: imageReference,
    nodeAgentSkuId: "batch.node.windows amd64"
);

CloudPool pool = batchClient.PoolOperations.CreatePool(
    poolId: "DotNetQuickstartPool",
    targetDedicatedComputeNodes: 2,
    virtualMachineSize: "STANDARD_A1_v2",
    virtualMachineConfiguration: vmConfiguration);
pool.Commit();

// Create a Batch job
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "DotNetQuickstartJob";
job.PoolInformation = new PoolInformation { PoolId = "DotNetQuickstartPool" };
job.Commit();

// Create a task to process each of the input files (uploaded to blob storage earlier)
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List<ResourceFile> { inputFiles[i] };

// Add all tasks to the job
batchClient.JobOperations.AddTask(JobId, tasks);

// Wait for tasks to complete (timeout after 30 mins)
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, TimeSpan.FromMinutes(30));

// Task standard output file
task.GetNodeFile(Constants.StandardOutFileName).ReadAsString();