We currently use the REST API (based on microsoft samples) to upload blobs blockwise from a .NET client profile machine. The REST API examples use the Azure storage account name and access key directly to construct the SharedKey entry in the request header. For production code, we'll need to calculate the SharedKey on our server, and deliver it for the client to use during the session.
Examples of SharedKey creation for blobs provide me with a Url plus query string that contains the access parameters.
My question: How do I get this Url/query string key format work in conjunction with the SharedKey header entry required by the Azure REST API?
Any pointers or tips greatly appreciated!
R
Here you go. Obviously a lot of improvement can be made to this code :) Give it a try. Do let me know if it works for you. I was able to upload a blob in development storage using the code below:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Web;
using System.Net;
using System.Collections.Specialized;
using System.Globalization;
namespace UploadBlobUsingSASUrl
{
class Program
{
//This is part of SAS signature (query string). We will construct the URI later using this.
private static string sasSignature = "sr=c&st=2012-08-16T14%3A38%3A48Z&se=2012-08-16T15%3A38%3A48Z&sp=w&sig=aNTLYQtwA1UmjG7j8Lg44t8YThL16FkNYBi54kl4ZKo%3D";
//Blob storage endpoint
private static string blobStorageEndpoint = "http://127.0.0.1:10000/devstoreaccount1";
//Blob container name
private static string blobContainerName = "[blob container name. SAS URI with Write permission must be created on this blob container]";
//File to upload
private static string fileToUpload = @"[Full path of the file you wish to upload]";
//This is the default block size (This application always assumes that a file will be split in blocks and then uploaded).
private static int blockSize = 256 * 1024;//256 KB
//Storage service version (Unless you're using latest SAS related changes in cloud storage, use this version). For development storage always use this version.
private static string x_ms_version = "2011-08-18";
//Template for put block list
private static string blockListTemplate = @"{0}";
// Template for block id (to be included in put block list template)
private static string blockIdTemplate = "{0}";
//We'll keep a list of block ids.
private static List blockIds = new List();
static void Main(string[] args)
{
FileInfo file = new FileInfo(fileToUpload);
long totalFileSize = file.Length;//Get the file size
long bytesFrom = 0;
long bytesRemaining = totalFileSize;
string blobName = file.Name;
//This is the base URI which will be used for put blocks and put block list operations.
//It is essentially would be something like "http://127.0.0.1:10000/devstoreaccount1/myblobcontainer/myblobname?sassignature"
string baseUri = string.Format("{0}/{1}/{2}?{3}", blobStorageEndpoint, blobContainerName, blobName, sasSignature);
int counter = 0;
//In this loop, we'll read file in chunks and try and upload one chunk at a time.
while (true)
{
int bytesToRead = blockSize;
if (bytesRemaining < blockSize)
{
bytesToRead = (int)bytesRemaining;
}
//Read the file in chunks
byte[] fileContents = ReadFile(fileToUpload, bytesFrom, bytesToRead);
bytesRemaining -= fileContents.Length;
bytesFrom += fileContents.Length;
//Create block id
string blockId = string.Format("Block-{0:D5}", counter);
//Append that to the block id list.
blockIds.Add(blockId);
//Now let's upload the block.
var isBlockUploaded = UploadBlock(baseUri, fileContents, blockId);
Console.WriteLine("Block Id: " + blockId + " Block Size: " + fileContents.Length + " Uploaded: " + isBlockUploaded);
counter++;
if (bytesRemaining <= 0)
{
break;
}
}
//All blocks uploaded, now let's commit the block list
var isBlockListCommitted = CommitBlockList(baseUri, blockIds);
Console.WriteLine("Is Block List Committed: " + isBlockListCommitted);
Console.WriteLine("Press any key to terminate the program ....");
Console.ReadLine();
}
/// <summary>
/// This function reads a chunk of the file and returns that as byte array.
/// </summary>
/// <param name="fileName"></param>
/// <param name="bytesFrom"></param>
/// <param name="bytesToRead"></param>
/// <returns></returns>
private static byte[] ReadFile(string fileName, long bytesFrom, int bytesToRead)
{
using (FileStream fs = new FileStream(fileName, FileMode.Open))
{
byte[] byteArray = new byte[bytesToRead];
fs.Seek(bytesFrom, SeekOrigin.Begin);
fs.Read(byteArray, 0, bytesToRead);
return byteArray;
}
}
/// <summary>
/// This function uploads a block.
/// </summary>
/// <param name="baseUri"></param>
/// <param name="blockContents"></param>
/// <param name="blockId"></param>
/// <returns></returns>
private static bool UploadBlock(string baseUri, byte[] blockContents, string blockId)
{
bool isBlockUploaded = false;
//Create request URI -
string uploadBlockUri = string.Format("{0}&comp=block&blockId={1}", baseUri, Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId)));
// Create request object
var request = (HttpWebRequest) HttpWebRequest.Create(uploadBlockUri);
NameValueCollection requestHeaders = new NameValueCollection();
var requestDate = DateTime.UtcNow;
//Add request headers. Please note that since we're using SAS URI, we don't really need "Authorization" header.
requestHeaders.Add("x-ms-date", string.Format(CultureInfo.InvariantCulture, "{0:R}", requestDate));
requestHeaders.Add("x-ms-version", x_ms_version);
request.Headers.Add(requestHeaders);
//Set content length header.
request.ContentLength = blockContents.Length;
//Set request HTTP method.
request.Method = "PUT";
// Send the request
using (Stream requestStream = request.GetRequestStream())
{
requestStream.Write(blockContents, 0, blockContents.Length);
}
// Get the response
using (var response = (HttpWebResponse)request.GetResponse())
{
isBlockUploaded = response.StatusCode.Equals(HttpStatusCode.Created);
}
return isBlockUploaded;
}
/// <summary>
/// This function commits the block list.
/// </summary>
/// <param name="baseUri"></param>
/// <param name="blockIds"></param>
/// <returns></returns>
private static bool CommitBlockList(string baseUri, List<string> blockIds)
{
bool isBlockListCommitted = false;
//Create the request payload
StringBuilder blockIdsPayload = new StringBuilder();
foreach (var blockId in blockIds)
{
blockIdsPayload.AppendFormat(blockIdTemplate, Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId)));
}
string putBlockListPayload = string.Format(blockListTemplate, blockIdsPayload.ToString());
// Create request URI
string putBlockListUrl = string.Format("{0}&comp=blocklist", baseUri);
// Create request object.
var request = (HttpWebRequest)HttpWebRequest.Create(putBlockListUrl);
NameValueCollection requestHeaders = new NameValueCollection();
//Add request headers. Please note that since we're using SAS URI, we don't really need "Authorization" header.
var requestDate = DateTime.UtcNow;
requestHeaders.Add("x-ms-date", string.Format(CultureInfo.InvariantCulture, "{0:R}", requestDate));
requestHeaders.Add("x-ms-version", x_ms_version);
byte[] requestPayload = Encoding.UTF8.GetBytes(putBlockListPayload);
//Set content length header.
request.ContentLength = requestPayload.Length;
request.Headers.Add(requestHeaders);
//Set request HTTP method.
request.Method = "PUT";
// Send the request
using (Stream requestStream = request.GetRequestStream())
{
requestStream.Write(requestPayload, 0, requestPayload.Length);
}
// Get the response
using (var response = (HttpWebResponse)request.GetResponse())
{
isBlockListCommitted = response.StatusCode.Equals(HttpStatusCode.Created);
}
return isBlockListCommitted;
}
}
}
I believe you need to generate Shared Access Signature (SAS) URL. Am I correct? When you generate the SAS URLs, the permissions are encoded in the URI itself so that you don't have to use the authorization key anymore.
For generating SAS, you may find these two links useful:
http://msdn.microsoft.com/en-us/library/windowsazure/hh508996
http://msdn.microsoft.com/en-us/library/windowsazure/ee395415
Here is my resulting code (minus some cleanup to do) as successfully tested in the emulation environment. First, the client-side code, which are adapted methods from the Microsoft REST API examples, BlobHelper.cs. Then the server-side code that delivers the Endpoint URL used by the client code. Thanks again for the tips! R
//
// Client side: The "Endpoint" used below is the Uri as returned from the Server-side code below.
//
//
public bool PutBlock(int blockId, string[] blockIds, byte[] value)
{
return Retry<bool>(delegate()
{
HttpWebResponse response;
try
{
SortedList<string, string> headers = new SortedList<string, string>();
byte[] blockIdBytes = BitConverter.GetBytes(blockId);
string blockIdBase64 = Convert.ToBase64String(blockIdBytes);
blockIds[blockId] = blockIdBase64;
// SharedAccessKey version.
//End result will look similar to this in Fiddler if correct:
//PUT http://127.0.0.1:10000/devstoreaccount1/uploads/aecdfa39-7eaa-474a-9333-ecf43e6a0508?st=2012-08-17T16%3A11%3A53Z&se=2012-08-17T16%3A51%3A53Z&sr=b&sp=rw&sig=2%2Fs0R1L78S55pW5o2WontVvlZypjkTriWoljnycPbFc%3D&comp=block&blockid=AAAAAA== HTTP/1.1
//
response = CreateRESTRequestDirectUtf8("PUT", "&comp=block&blockid=" + blockIdBase64, value, headers).GetResponse() as HttpWebResponse;
response.Close();
return true;
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError &&
ex.Response != null &&
(int)((HttpWebResponse)ex.Response).StatusCode == 409)
return false;
throw;
}
});
}
///<summary>
/// Put block list - complete creation of blob based on uploaded content.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="blob">The BLOB.</param>
/// <param name="blockIds">The block ids.</param>
/// <returns></returns>
public bool PutBlockList(string[] blockIds)
{
return Retry<bool>(delegate()
{
HttpWebResponse response;
try
{
StringBuilder content = new StringBuilder();
content.Append("<?xml version=\"1.0\" encoding=\"utf-8\"?>");
content.Append("<BlockList>");
for (int i = 0; i < blockIds.Length; i++)
{
content.Append("<Latest>" + blockIds[i] + "</Latest>");
}
content.Append("</BlockList>");
response = CreateRESTRequest("PUT", "&comp=blocklist", content.ToString(), null).GetResponse() as HttpWebResponse;
response.Close();
return true;
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError &&
ex.Response != null &&
(int)(ex.Response as HttpWebResponse).StatusCode == 409)
return false;
throw;
}
});
}
/// <summary>
/// Construct and issue a REST request and return the response.
/// </summary>
/// <param name="method">The method.</param>
/// <param name="resource">The resource.</param>
/// <param name="requestBody">The request body.</param>
/// <param name="headers">The headers.</param>
/// <param name="ifMatch">If match.</param>
/// <param name="md5">The MD5.</param>
/// <returns></returns>
public HttpWebRequest CreateRESTRequest(string method, string resource, string requestBody = null, SortedList<string, string> headers = null,
string ifMatch = "", string md5 = "")
{
byte[] byteArray = null;
DateTime now = DateTime.UtcNow;
Uri uri = new Uri(Endpoint + resource);
HttpWebRequest request = HttpWebRequest.Create(uri) as HttpWebRequest;
request.Method = method;
request.ContentLength = 0;
request.Headers.Add("x-ms-date", now.ToString("R", System.Globalization.CultureInfo.InvariantCulture));
request.Headers.Add("x-ms-version", "2009-09-19"); //2009-09-19, 2011-08-18
if (IsTableStorage)
{
request.ContentType = "application/atom+xml";
request.Headers.Add("DataServiceVersion", "1.0;NetFx");
request.Headers.Add("MaxDataServiceVersion", "1.0;NetFx");
}
if (headers != null)
{
foreach (KeyValuePair<string, string> header in headers)
{
request.Headers.Add(header.Key, header.Value);
}
}
if (!String.IsNullOrEmpty(requestBody))
{
request.Headers.Add("Accept-Charset", "UTF-8");
byteArray = Encoding.UTF8.GetBytes(requestBody);
request.ContentLength = byteArray.Length;
}
// We now get our SharedAccessKey from the server
//request.Headers.Add("Authorization", AuthorizationHeader(method, now, request, ifMatch, md5));
if (!String.IsNullOrEmpty(requestBody))
{
request.GetRequestStream().Write(byteArray, 0, byteArray.Length);
}
return request;
}
/// <summary>
/// Creates the REST request direct UTF8.
/// </summary>
/// <param name="method">The method.</param>
/// <param name="resource">The resource.</param>
/// <param name="requestBodyUtf8">The request body UTF8.</param>
/// <param name="headers">The headers.</param>
/// <param name="ifMatch">If match.</param>
/// <param name="md5">The MD5.</param>
/// <returns></returns>
private HttpWebRequest CreateRESTRequestDirectUtf8(string method, string resource, byte[] requestBodyUtf8, SortedList<string, string> headers = null, string ifMatch = "", string md5 = "")
{
//byte[] byteArray = null;
DateTime now = DateTime.UtcNow;
Uri uri = new Uri(Endpoint + resource);
HttpWebRequest request = HttpWebRequest.Create(uri) as HttpWebRequest;
request.Method = method;
request.ContentLength = 0;
request.Headers.Add("x-ms-date", now.ToString("R", System.Globalization.CultureInfo.InvariantCulture));
request.Headers.Add("x-ms-version", "2009-09-19"); //2009-09-19, 2011-08-18
if (IsTableStorage)
{
request.ContentType = "application/atom+xml";
request.Headers.Add("DataServiceVersion", "1.0;NetFx");
request.Headers.Add("MaxDataServiceVersion", "1.0;NetFx");
}
// Additional headers can be passed in as a formal parameter:
if (headers != null)
{
foreach (KeyValuePair<string, string> header in headers)
{
request.Headers.Add(header.Key, header.Value);
}
}
if (requestBodyUtf8 != null)
{
request.Headers.Add("Accept-Charset", "UTF-8");
request.ContentLength = requestBodyUtf8.Length;
}
// We now get our SharedAccessKey from the server
//request.Headers.Add("Authorization", AuthorizationHeader(method, now, request, ifMatch, md5));
if (requestBodyUtf8 != null)
{
request.GetRequestStream().Write(requestBodyUtf8, 0, requestBodyUtf8.Length);
}
return request;
}
//
// Server side: The returned Uri here is the "Endpoint" used in the client code.
//
/// <summary>
/// Gets the blob-level shared access signature for the named blob
/// </summary>
/// <param name="blobName">The unique blob name Guid.</param>
/// <returns>The fully qualified Azure Shared Access Signature Query String to be used in azure upload connections</returns>
public Uri GetBlobUploadUrl(Guid blobName)
{
string containerName = BlobContainerName;
const string permissions = "rw";
string sharedAccessSignature = CreateSharedAccessSignature(containerName, blobName.ToString(), permissions);
string urlPath;
if (Microsoft.WindowsAzure.ServiceRuntime.RoleEnvironment.IsEmulated)
{ // Emulation environment
urlPath = String.Format("{0}/{1}/{2}{3}", _blobEndpoint, containerName, blobName, sharedAccessSignature);
}
else
{ // Cloud
urlPath = String.Format("{0}{1}/{2}{3}", _blobEndpoint, containerName, blobName, sharedAccessSignature);
}
Uri uri = new Uri(urlPath);
return uri;
}
/// <summary>
/// Creates a blob-level shared access signature.
/// </summary>
/// <param name="containerName">The blob container name.</param>
/// <param name="blobName">The blob name, a unique ID which will be passed back to the client.</param>
/// <param name="permissions">String of access levels, "r" = read, "w" = write "rw" = both etc.</param>
/// <returns>The fully qualified Azure Shared Access Signature Query String</returns>
private string CreateSharedAccessSignature(string containerName, string blobName, string permissions)
{
// SAS without stored container policy
const string iso8061Format = "{0:yyyy-MM-ddTHH:mm:ssZ}";
DateTime startTime = DateTime.UtcNow.AddMinutes(-10d); //UtcNow;
DateTime expiryTime = startTime.AddMinutes(40d);
string start = string.Format(iso8061Format, startTime);
string expiry = string.Format(iso8061Format, expiryTime);
string stringToSign = string.Format("{0}\n{1}\n{2}\n/{3}/{4}/{5}\n", permissions, start, expiry, _accountName, containerName, blobName);
// SAS with stored container policy
//string stringToSign = String.Format("\n\n\n/{0}/{1}\n{2}", accountName, containerName, policyId);
string rawSignature = String.Empty;
Byte[] keyBytes = Convert.FromBase64String(_accountKey);
using (HMACSHA256 hmacSha256 = new HMACSHA256(keyBytes))
{
Byte[] utf8EncodedStringToSign = System.Text.Encoding.UTF8.GetBytes(stringToSign);
Byte[] signatureBytes = hmacSha256.ComputeHash(utf8EncodedStringToSign);
rawSignature = Convert.ToBase64String(signatureBytes);
}
string sharedAccessSignature = String.Format("?st={0}&se={1}&sr=b&sp={2}&sig={3}", Uri.EscapeDataString(start), Uri.EscapeDataString(expiry), permissions, Uri.EscapeDataString(rawSignature));
//
// End result will look like this in Fiddler if correct:
//PUT http://127.0.0.1:10000/devstoreaccount1/uploads/aecdfa39-7eaa-474a-9333-ecf43e6a0508?st=2012-08-17T16%3A11%3A53Z&se=2012-08-17T16%3A51%3A53Z&sr=b&sp=rw&sig=2%2Fs0R1L78S55uW5o2WontVvrZypckTriWoijnyrPbFc%3D&comp=block&blockid=AAAAAA== HTTP/1.1
//
return sharedAccessSignature;
}
