Poll task status

Structured Data File (SDF) generation can last anywhere from a few seconds to hours. The task is done when the done field in the Operation is True.

An Operation is identified by a name of the format /sdfdownloadtasks/operations/{id}. Regularly retrieve the Operation with this name using sdfdownloadtasks.operations.get to see if the task is finished. This process is known as "polling."

If done is True, either the response or error field will be set. A failed task sets the error field. The assigned Status object provides details about the failure. A successful task sets the response field. Download the resulting SDFs from response.resourceName.

Inefficient polling can consume a lot of API request quota. Limit retries and conserve quota using exponential backoff.

Here's how to poll a SDF download task using exponential backoff:

Java

// Provide the name of the sdfdownloadtask operation.
String operationName = operation-name;

// Configure the Operations.get request.
Sdfdownloadtasks.Operations.Get operationRequest =
    service.sdfdownloadtasks().operations().get(operationName);

// Configure exponential backoff for checking the status of our operation.
ExponentialBackOff backOff =
    new ExponentialBackOff.Builder()
        .setInitialIntervalMillis(5000) // setting initial interval to five seconds
        .setMaxIntervalMillis(5000 * 60) // setting max interval to five minutes
        .setMaxElapsedTimeMillis(5000 * 60 * 60) // setting max elapsed time to five hours
        .build();
long backoffMillis = 0;

// Create default operation variable.
Operation operation = new Operation();

do {
  // Sleep before retrieving operation again.
  Thread.sleep(backoffMillis);

  // Retrieve operation.
  operation = operationRequest.execute();

  // If operation is not done, calculate next sleep period.
  if (operation.getDone() == null) {
    backoffMillis = backOff.nextBackOffMillis();
    if (backoffMillis == ExponentialBackOff.STOP) {
      System.out.printf("The operation has taken more than five hours to complete.%n");
      break;
    }
    System.out.printf(
        "Operation %s is still running, sleeping for %s milliseconds.%n",
        operationName, backoffMillis);
  }
} while (operation.getDone() == null);

// Check whether operation finished with an error, or is completed and ready to download.
if (operation.getDone() != null) {
  if (operation.getError() != null) {
    System.out.printf(
        "The operation finished in error with code %s: %s%n",
        operation.getError().getCode(), operation.getError().getMessage());
  } else {
    System.out.printf(
        "The operation completed successfully. Resource %s was created.%n",
        operation.getResponse().get("resourceName").toString());
  }
}

Python

# Provide the name of the sdfdownloadtask operation.
operation_name = operation-name

# Set the following values that control retry behavior while the operation
# is running.
# Minimum amount of time between polling requests. Defaults to 5 seconds.
min_retry_interval = 5
# Maximum amount of time between polling requests. Defaults to 5 minutes.
max_retry_interval = 5 * 60
# Maximum amount of time to spend polling. Defaults to 5 hours.
max_retry_elapsed_time = 5 * 60 * 60

def next_sleep_interval(previous_sleep_interval):
  """Calculates the next sleep interval based on the previous."""
  min_interval = previous_sleep_interval or min_retry_interval
  max_interval = previous_sleep_interval * 3 or min_retry_interval
  return min(max_retry_interval, random.randint(min_interval, max_interval))

# Configure the sdfdownloadtasks.operations.get request.
get_request = service.sdfdownloadtasks().operations().get(name=operation_name)

sleep = 0
start_time = time.time()

while True:
  # Retrieve the operation.
  operation = get_request.execute()

  if "done" in operation:
    if "error" in operation:
      print(
          f"The operation finished in error with code "
          f'{operation["error"]["code"]}: {operation["error"]["message"]}'
      )
    else:
      print(
          f"The operation completed successfully. The resulting files can be "
          f'downloaded at {operation["response"]["resourceName"]}.'
      )
    break
  elif time.time() - start_time > max_retry_elapsed_time:
    print("SDF generation deadline exceeded.")
    break

  sleep = next_sleep_interval(sleep)
  print(
      f"Operation {operation_name} is still running, sleeping for "
      f"{sleep} seconds."
  )
  time.sleep(sleep)

PHP

// The following values control retry behavior while the task is processing.
// Minimum amount of time between polling requests. Defaults to 5 seconds.
$minRetryInterval = 5;
// Maximum amount of time between polling requests. Defaults to 5 minutes.
$maxRetryInterval = 300;
// Maximum amount of time to spend polling. Defaults to 5 hours.
$maxRetryElapsedTime = 18000;

// Provide the name of the sdfdownloadtask operation.
$operationName = operation-name;

// Poll operation.
$sleep = 0;
$startTime = time();
do {
    try {
        // Call the API, retrieving the SDF Download Task.
        $operation = $this->service->sdfdownloadtasks_operations->get(
            $operationName
        );
    } catch (\Exception $e) {
        $this->renderError($e);
        return;
    }

    // Check if retrieved operation is finished.
    if ($operation->getDone() === true) {
        if($operation->getError() !== null) {
            $error = $operation->getError();
            printf(
                'The operation finished in error with code %s: %s<br>',
                $error->setCode(),
                $error->setMessage()
            );
            break;
        } else {
            $response = $operation->getResponse();
            printf(
                'The operation completed successfully. Resource %s '
                    . 'was created.<br>',
                $response['resourceName']
            );
            break;
        }
    } elseif (time() - $startTime > self::MAX_RETRY_ELAPSED_TIME) {
        print '<p>SDF download task processing deadline exceeded</p>';
        break;
    }

    // Generate next sleep interval.
    $minInterval = max($minRetryInterval, $sleep);
    $maxInterval = max(
        $minRetryInterval,
        $sleep * 3
    );
    $sleep = min($maxRetryInterval, rand($minInterval, $maxInterval));

    // Sleep a determined amount of time before retrieving the operation
    // again.
    printf(
        'Operation %s is still running, sleeping for %d seconds<br>',
        $operationName,
        $sleep
    );
    sleep($sleep);
} while (true);