message RolloutRequest {
    // Index of rollout
    int32 rollout_index = 1;
    // Random seed from the master
    int32 env_seed = 2;
    // The weights of the neural network policy
    Repeated double policy_weights = 3;
    // Whether this request is evaluation
    bool evaluate = 4;
}

message RolloutResponse {
    // Index of rollout
    int32 rollout_index = 1;
    // The reward collected in the rollout
    double rollout_rewards = 2;
}

service Rollout {
  // The RPC service for policy evaluation
  rpc RolloutWithParameter(RolloutRequest) returns (RolloutResponse) {}
}

  GCLOUD_PROJECT={your-project-id}

# Create a cluster.
gcloud container clusters create es-cluster \
--zone=us-central1-a \
--machine-type=n1-standard-64 \
--max-nodes=20 \
--min-nodes=16 \
--num-nodes=17 \
--enable-autoscaling \
--project ${GCLOUD_PROJECT}

  # Step 1: Build a container image for the controller and the workers.
gcloud builds submit \
--tag gcr.io/${GCLOUD_PROJECT}/es-on-gke:1.0 . \
--timeout 3600 \
--project ${GCLOUD_PROJECT}

# Step 2: Replace the ${GCLOUD_PROJECT} in the YAML file and run the command.
kubectl apply -f deploy_workers.yaml

# Step 3: When all the workers are started and running,
#         replace the ${GCLOUD_PROJECT} in the YAML file and run the command.
kubectl apply -f deploy_master.yaml

  POD_NAME=$(kubectl get pod | grep es-master | awk '{print $1}')

# Download reward vs time plot.
kubectl cp $POD_NAME:/var/log/es/log/reward_vs_time.png $HOME/

# Download reward vs iteration plot.
kubectl cp $POD_NAME:/var/log/es/log/reward_vs_iteration.png $HOME/

# Download best model so far.
kubectl cp $POD_NAME:/var/log/es/log/best_model.npz $HOME/

# Download model at iteration 1000.
kubectl cp $POD_NAME:/var/log/es/log/model_1000.npz $HOME/

# Download all test scores.
kubectl cp $POD_NAME:/var/log/es/log/scores.csv $HOME/

  # Train locally.
# E.g. bash ./train_local.sh -c configs/BipedalWalkerHardcore.gin -n 10
bash ./train_local.sh -c {path-to-config-file} -n {number-of-workers}

# Test locally.
# E.g. python test.py --checkpoint=1000 --logdir=./log
python test.py --checkpoint={checkpoint} --logdir={path-to-log-directory}

  SELECT
  w1mpro_ep,
  mjd,
  load_id,
  frame_id
FROM
  `bigquery-public-data.wise_all_sky_data_release.mep_wise`
WHERE
  source_id_mf='3425p651_ac51-032187'
ORDER BY
  mjd ASC

  CREATE TEMP FUNCTION
  ArcSecondDistance(p1 GEOGRAPHY, p2 GEOGRAPHY,
    d FLOAT64) AS (ST_DISTANCE(p1,p2) < d * 30.8874796235);
SELECT
  source_id_mf,
  point
FROM
  `bigquery-public-data.wise_all_sky_data_release.mep_wise`
WHERE
  ArcSecondDistance(point,
    ST_GEOGPOINT(201.5, -2.6),60)
  AND ST_CONTAINS(
   ST_GEOGFROMTEXT('Polygon((201.00 -3.10,201.00 -2.10,202.00 -2.10,202.00 -3.10, 201.00 -3.10))'),point)

  package main

import (
        "context"
        "encoding/base64"
        "fmt"
        "log"
        "net/http"
        "os"
        "time"

        "cloud.google.com/go/storage"
        "github.com/google/uuid"
        "golang.org/x/oauth2/google"
        iam "google.golang.org/api/iam/v1"
)

var (
        // iamService is a client for calling the signBlob API.
        iamService *iam.Service

        // serviceAccountName represents Service Account Name.
        // See more details: https://cloud.google.com/iam/docs/service-accounts
        serviceAccountName string

        // serviceAccountID follows the below format.
        // "projects/%s/serviceAccounts/%s"
        serviceAccountID string

        // uploadableBucket is the destination bucket.
        // All users will upload files directly to this bucket by using generated Signed URL.
        uploadableBucket string
)

func signHandler(w http.ResponseWriter, r *http.Request) {
        // Accepts only POST method.
        // Otherwise, this handler returns 405.
        if r.Method != "POST" {
                w.Header().Set("Allow", "POST")
                http.Error(w, "Only POST is supported", http.StatusMethodNotAllowed)
                return
        }

        ct := r.FormValue("content_type")
        if ct == "" {
                http.Error(w, "content_type must be set", http.StatusBadRequest)
                return
        }

        // Generates an object key for use in new Cloud Storage Object.
        // It's not duplicate with any object keys because of UUID.
        key := uuid.New().String()
        if ext := r.FormValue("ext"); ext != "" {
                key += fmt.Sprintf(".%s", ext)
        }

        // Generates a signed URL for use in the PUT request to GCS.
        // Generated URL should be expired after 15 mins.
        url, err := storage.SignedURL(uploadableBucket, key, &storage.SignedURLOptions{
                GoogleAccessID: serviceAccountName,
                Method:         "PUT",
                Expires:        time.Now().Add(15 * time.Minute),
                ContentType:    ct,
                // To avoid management for private key, use SignBytes instead of PrivateKey.
                // In this example, we are using the `iam.serviceAccounts.signBlob` API for signing bytes.
                // If you hope to avoid API call for signing bytes every time,
                // you can use self hosted private key and pass it in Privatekey.
                SignBytes: func(b []byte) ([]byte, error) {
                        resp, err := iamService.Projects.ServiceAccounts.SignBlob(
                                serviceAccountID,
                                &iam.SignBlobRequest{BytesToSign: base64.StdEncoding.EncodeToString(b)},
                        ).Context(r.Context()).Do()
                        if err != nil {
                                return nil, err
                        }
                        return base64.StdEncoding.DecodeString(resp.Signature)
                },
        })
        if err != nil {
                log.Printf("sign: failed to sign, err = %v\n", err)
                http.Error(w, "failed to sign by internal server error", http.StatusInternalServerError)
                return
        }
        w.WriteHeader(http.StatusOK)
        fmt.Fprintln(w, url)
}

func main() {
        cred, err := google.DefaultClient(context.Background(), iam.CloudPlatformScope)
        if err != nil {
                log.Fatal(err)
        }
        iamService, err = iam.New(cred)
        if err != nil {
                log.Fatal(err)
        }

        uploadableBucket = os.Getenv("UPLOADABLE_BUCKET")
        serviceAccountName = os.Getenv("SERVICE_ACCOUNT")
        serviceAccountID = fmt.Sprintf(
                "projects/%s/serviceAccounts/%s",
                os.Getenv("GOOGLE_CLOUD_PROJECT"),
                serviceAccountName,
        )

        http.HandleFunc("/sign", signHandler)
        log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), nil))
}

  package main

import (
        "bytes"
        "fmt"
        "io/ioutil"
        "log"
        "net/http"
        "net/url"
        "strings"
)

const signerUrl = "<APPENGINE_URL>"

func getSignedURL(target string, values url.Values) (string, error) {
        resp, err := http.PostForm(target, values)
        if err != nil {
                return "", err
        }
        defer resp.Body.Close()
        b, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                return "", err
        }
        return strings.TrimSpace(string(b)), nil
}

func main() {
        // Get signed url from the API server hosted on App Engine.
        u, err := getSignedURL(signerUrl, url.Values{"content_type": {"image/png"}})
        if err != nil {
                log.Fatal(err)
        }
        fmt.Printf("Signed URL here: %q\n", u)

        b, err := ioutil.ReadFile("/path/to/sample.png")
        if err != nil {
                log.Fatal(err)
        }
        // Generates *http.Request to request with PUT method to the Signed URL.
        req, err := http.NewRequest("PUT", u, bytes.NewReader(b))
        if err != nil {
                log.Fatal(err)
        }
        req.Header.Add("Content-Type", "image/png")
        client := new(http.Client)
        resp, err := client.Do(req)
        if err != nil {
                log.Fatal(err)
        }
        fmt.Println(resp)
}

  {
  "rule": [
    {
      "action": {"type": "Delete"},
      "condition": {"age": 1}
    }
  ]
}

  REGION="REGION"
PROJECT_ID="PROJECT_ID"
UPLOADABLE_BUCKET="UPLOADABLE_BUCKET"
DISTRIBUTION_BUCKET="DISTRIBUTION_BUCKET"
LIFECYCLE_POLICY_FILE="/path/to/lifecycle.json"

# Creates the uploadable bucket
gsutil mb -p $PROJECT_ID -l $REGION --retention 900s gs://$UPLOADABLE_BUCKET
# Creates the bucket for distribution
gsutil mb -p $PROJECT_ID -l $REGION gs://$DISTRIBUTION_BUCKET
# Set lifecyle for the uploadable bucket
gsutil lifecycle set $LIFECYCLE_POLICY_FILE gs://$UPLOADABLE_BUCKET

  package function

import (
        "context"
        "errors"
        "fmt"
        "image"
        "image/gif"
        "image/jpeg"
        "image/png"
        "log"

        "cloud.google.com/go/storage"
        vision "cloud.google.com/go/vision/apiv1"
        "golang.org/x/xerrors"
        pb "google.golang.org/genproto/googleapis/cloud/vision/v1"
)

type GCSEvent struct {
        Bucket string `json:"bucket"`
        Name   string `json:"name"`
}

var retryableError = xerrors.New("upload: retryable error")

func validate(ctx context.Context, obj *storage.ObjectHandle) error {
        attrs, err := obj.Attrs(ctx)
        if err != nil {
                return xerrors.Errorf("upload: failed to get object attributes %q : %w",
                        obj.ObjectName(), retryableError)
        }
        // You can enlarge maximum size up to 20MB by modifying this line. 
        if attrs.Size >= 1024*100 {
                return fmt.Errorf("upload: image file is too large, got = %d", attrs.Size)
        }
        // Validates obj and returns true if it conforms supported image formats.
        if err := validateMIMEType(ctx, attrs, obj); err != nil {
                return err
        }
        // Validates obj by calling Vision API.
        return validateByVisionAPI(ctx, obj)
}

func validateMIMEType(ctx context.Context, attrs *storage.ObjectAttrs, obj *storage.ObjectHandle) error {
        r, err := obj.NewReader(ctx)
        if err != nil {
                return xerrors.Errorf("upload: failed to open new file %q : %w",
                        obj.ObjectName(), retryableError)
        }
        defer r.Close()
        if _, err := func(ct string) (image.Image, error) {
                switch ct {
                case "image/png":
                        return png.Decode(r)
                case "image/jpeg", "image/jpg":
                        return jpeg.Decode(r)
                case "image/gif":
                        return gif.Decode(r)
                default:
                        return nil, fmt.Errorf("upload: unsupported MIME type, got = %q", ct)
                }
        }(attrs.ContentType); err != nil {
                return err
        }
        return nil
}

// validateByVisionAPI uses Safe Search Detection provided by Cloud Vision API.
// See more details: https://cloud.google.com/vision/docs/detecting-safe-search
func validateByVisionAPI(ctx context.Context, obj *storage.ObjectHandle) error {
        client, err := vision.NewImageAnnotatorClient(ctx)
        if err != nil {
                return xerrors.Errorf(
                        "upload: failed to create a ImageAnnotator client, error = %v : %w",
                        err,
                        retryableError,
                )
        }
        ssa, err := client.DetectSafeSearch(
                ctx,
                vision.NewImageFromURI(fmt.Sprintf("gs://%s/%s", obj.BucketName(), obj.ObjectName())),
                nil,
        )
        if err != nil {
                return xerrors.Errorf(
                        "upload: failed to detect safe search, error = %v : %w",
                        err,
                        retryableError,
                )
        }
        // Returns an unretryable error if there is any possibility of inappropriate image.
        // Likelihood has been defined in the following:
        // https://github.com/google/go-genproto/blob/5fe7a883aa19554f42890211544aa549836af7b7/googleapis/cloud/vision/v1/image_annotator.pb.go#L37-L50
        if ssa.Adult >= pb.Likelihood_POSSIBLE ||
                ssa.Medical >= pb.Likelihood_POSSIBLE ||
                ssa.Violence >= pb.Likelihood_POSSIBLE ||
                ssa.Racy >= pb.Likelihood_POSSIBLE {
                return errors.New("upload: exceeds the prescribed likelihood")
        }
        return nil
}

// distributionBucket is the distribution bucket.
// It's used for distributing all of passed files.
// TODO: This value MUST be updated before deploying this function.
const distributionBucket = "DISTRIBUTION_BUCKET"

// UplaodImage validates the object and copy it into the distribution bucket.
func UploadImage(ctx context.Context, e GCSEvent) error {
        client, err := storage.NewClient(ctx)
        if err != nil {
                return fmt.Errorf("upload: failed to construct a client, error = %v", err)
        }
        defer client.Close()

        dst := client.Bucket(distributionBucket).Object(e.Name)
        _, err = dst.Attrs(ctx)
        // Avoid proceeding if the object has been copied to destination.
        if err == nil {
                log.Printf("upload: %s has already been copied to destination\n", e.Name)
                return nil
        }
        // Return retryable error as there is a possibility that object does not temporarily exist.
        if err != storage.ErrObjectNotExist {
                return err
        }
        src := client.Bucket(e.Bucket).Object(e.Name)
        if err := validate(ctx, src); err != nil {
                if xerrors.Is(err, retryableError) {
                        return err
                }
                log.Println(err)
                return nil
        }
        // Returns an error if the copy operation failed.
        // Will retry the same processing later.
        if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
                return err
        }

        return nil
}

  gcloud functions deploy UploadImage --runtime go111 --trigger-resource [UPLOADABLE_BUCKET] --trigger-event google.storage.object.finalize --retry