b2_upload_part

Code

FILE_NAME='./bigfile.dat'
FILE_SIZE=`stat -s $FILE_NAME | awk '{print $'8'}' | sed 's/st_size=\([0-9]*\)/\1/g'`

# Prepare the large file be spliting it into chunks
split -b 100000000 $FILE_NAME bz_chunk_
FILE_CHUNK_NAMES=(`ls -1 bz_chunk_*`)

# Upload file
UPLOAD_URL=''; # Provided by b2_get_upload_part_url
UPLOAD_AUTHORIZATION_TOKEN=''; # Provided by b2_get_upload_part_url
let MINIMUM_PART_SIZE=(100 * 1000 * 1000); # Provided by b2_authorize_account. Default 100 MB
TOTAL_BYTES_SENT=0;
BYTES_SENT=$MINIMUM_PART_SIZE;
PART_NO=1;
PART_SHA1_ARRAY=();
while [ $TOTAL_BYTES_SENT -lt $FILE_SIZE ]; do
    if [ $(($FILE_SIZE-$TOTAL_BYTES_SENT)) -lt $MINIMUM_PART_SIZE ]; then
	    let BYTES_SENT=$(($FILE_SIZE - $TOTAL_BYTES_SENT));
    fi
    let IDX=PART_NO-1;
    CHUNK_FILE=${FILE_CHUNK_NAMES[IDX]};	
    CHUNK_SHA1=`openssl sha1 $CHUNK_FILE | awk '{print $'2'}'`;
    PART_SHA1_ARRAY=(${PART_SHA1_ARRAY[@]} $CHUNK_SHA1);
    curl \
        -H "Authorization: $UPLOAD_AUTHORIZATION_TOKEN" \
        -H "X-Bz-Part-Number: $PART_NO" \
        -H "X-Bz-Content-Sha1: ${PART_SHA1_ARRAY[$IDX]}" \
        -H "Content-Length: $BYTES_SENT" \
        --data-binary "@$CHUNK_FILE" \
        $UPLOAD_URL
    let TOTAL_BYTES_SENT+=$BYTES_SENT;
    let PART_NO+=1;
done

Output

Assuming ~208 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Part 3:
{
  "contentLength": 8158542,
  "contentSha1": "4546018a346df683acc9a3367977de4cc8024965",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 3
}

Code

import java.io.*;
import java.util.*;
import javax.json.*;
import java.math.BigInteger;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;      

String uploadUrl = ""; // Provided by b2_get_upload_part_url
String uploadAuthorizationToken = ""; // Provided by b2_get_upload_part_url
long minimumPartSize = 0; // Provided by b2_authorize_account

// Open the file and
File localFile = new File("bigfile.dat");
long localFileSize = localFile.length();
long totalBytesSent = 0;
long bytesSentForPart = minimumPartSize;
byte[] buf = new byte[(int)minimumPartSize];
ArrayList<String> partSha1Array = new ArrayList<String>();
int partNo = 1;
while (totalBytesSent < localFileSize) {
    
    // Determine how many bytes we are going to send
    if ((localFileSize - totalBytesSent) < minimumPartSize) {
        bytesSentForPart = (localFileSize - totalBytesSent);
    }

    try {
        // Get the the SHA1 of the one parts we are uploading
        FileInputStream fileInputStream = new FileInputStream(localFile);
        fileInputStream.skip(totalBytesSent);
        int numBytesReadIntoBuff = fileInputStream.read(buf, 0, (int) bytesSentForPart);
        fileInputStream.close();
        MessageDigest sha1Digest = MessageDigest.getInstance("SHA-1");
        sha1Digest.update(buf, 0, (int) bytesSentForPart);
        byte[] sha1Buf = sha1Digest.digest();
        String hexSha1 = String.format("%040x", new BigInteger(1, sha1Buf));
        partSha1Array.add(hexSha1);

        // Send it over the wire
        URL url = new URL(uploadUrl);
        connection = (HttpURLConnection) url.openConnection();
        connection.setRequestMethod("POST");
        connection.setRequestProperty("Authorization", uploadAuthorizationToken);
        connection.setRequestProperty("X-Bz-Part-Number", new Integer(partNo).toString());
        connection.setRequestProperty("Content-Length", new Long(bytesSentForPart).toString());
        connection.setRequestProperty("X-Bz-Content-Sha1", partSha1Array.get(partNo - 1));
        connection.setDoOutput(true);
        DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream());
        dataOutputStream.write(buf, 0, (int) bytesSentForPart);
        dataOutputStream.close();
        String jsonResponse = myInputStreamReader(connection.getInputStream());
        System.out.println(jsonResponse);

    } catch (FileNotFoundException fnfe) {
        fnfe.printStackTrace();
    } catch (IOException iox) {
        iox.printStackTrace();
    } catch (NoSuchAlgorithmException nsae) {
        nsae.printStackTrace();
    } finally {
        connection.disconnect();
    }

    // Get read for the next iteration of the loop
    totalBytesSent = totalBytesSent + bytesSentForPart;
    partNo++;
}

// Input stream reader example. 
static public String myInputStreamReader(InputStream in) throws IOException {
    InputStreamReader reader = new InputStreamReader(in);
    StringBuilder sb = new StringBuilder();
    int c = reader.read();
    while (c != -1) {
        sb.append((char)c);
        c = reader.read();
    }
    reader.close();
    return sb.toString();
}

Output

Assuming ~208 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Part 3:
{
  "contentLength": 8158542,
  "contentSha1": "4546018a346df683acc9a3367977de4cc8024965",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 3
}

Code

import base64
import json
import urllib2
import os
import hashlib

upload_authorization_token = "" # Provided by b2_get_upload_part_url
local_file = "/path/to/file/on/disk"
local_file_size = os.stat(local_file).st_size
size_of_part = minimum_part_size
total_bytes_sent = 0
part_no = 1
part_sha1_array = []
while (total_bytes_sent < local_file_size):
	if ((local_file_size - total_bytes_sent) < minimum_part_size):
		size_of_part = local_file_size - total_bytes_sent
	filed = open(local_file)
	filed.seek(total_bytes_sent)
	file_data = filed.read(size_of_part)
	filed.close()
	sha1_digester = hashlib.new('SHA1')
	sha1_digester.update(file_data)
	sha1_str = sha1_digester.hexdigest()
	part_sha1_array.append(sha1_str)
	request = urllib2.Request(
    	upload_url,
    	file_data,
    	headers = { 'Authorization': upload_authorization_token, 
    	            'X-Bz-Part-Number': part_no, 
    	            'Content-Length':  size_of_part, 
    	            'X-Bz-Content-Sha1': sha1_str }
    	            )      
	response = urllib2.urlopen(request)
	response_data = json.loads(response.read())
	response.close()
	print response_data
	total_bytes_sent = total_bytes_sent + size_of_part
	part_no += 1

Output

Assuming ~208 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Part 3:
{
  "contentLength": 8158542,
  "contentSha1": "4546018a346df683acc9a3367977de4cc8024965",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 3
}

Code

import Foundation

// For this example we are assuming a file size of 200 MB consisting of
// the letter 'B'. In the example, we will upload two 100 MB parts.
// The sample code will acquire two upload URLs which are required when
// uploading file parts simultaneously (see the Large Files documentation
// for more detail).
//
// Other assumptions:
//
//   SHA1 - We assume you have a way to produce a SHA1 from the given
//          data objects.  If you are using the Apple platform please take a
//          look at importing <CommonCrypto/CommonCrypto.h>.  If you are
//          using Linux OpenSSL or <blah> can also generate a SHA1.
//
//   Hex String - We assume that you have a routine to convert a Data
//          object to a Hex string.  This is will be used to create a
//          Hex string representation of the upload parts.
//

// Generate the parts of the large 'B' file.  Typically you would read
// the file off of disk.

// Call b2_authorize_account you need this for an authorization token
let authInfo = "" // Obtained by b2_authorize_account
let authToken = "" // Obtained by b2_authorize_account

// Call b2_start_large_file you need this for the file ID which is
// required by b2_get_upload_part_url.

// Since we are simultaneously uploading both parts we need an upload
// authorization token and upload url for each part.  See
// b2_get_upload_part_url documentation for more information.
//
// Call b2_get_upload_part_url to get a URL and auth token for the 1st part
let dataPart1 = Data(repeating: UInt8(ascii: "B"), count: (100 * 1024 * 1024))
let dataPart1ShaHexStr = ""

// Prepare the upload request for part 1
let uploadUrlForPart1 = URL(string: "<upload part url obtained from b2_get_upload_part_url>")!
let uploadAuthTokenForPart1 = "<upload part authorization token obtained from b2_get_upload_part_url>"
var requestForPart1 = URLRequest(url: uploadUrlForPart1)
requestForPart1.httpMethod = "POST"
requestForPart1.addValue(uploadAuthTokenForPart1, forHTTPHeaderField: "Authorization")
requestForPart1.addValue(String(1), forHTTPHeaderField: "X-Bz-Part-Number")
requestForPart1.addValue(String(dataPart1.count), forHTTPHeaderField: "Content-Length")
requestForPart1.addValue(dataPart1ShaHexStr, forHTTPHeaderField: "X-Bz-Content-Sha1")

// Create the task
let taskForPart1 = URLSession.shared.uploadTask(with: requestForPart1, from: dataPart1) { (data, response, error) in
	if let jsonData = data {
		let json = String(data: jsonData, encoding: .utf8)
		print("\(json!)")
	}
}

// Start the task
taskForPart1.resume()

// Call b2_get_upload_part_url to get a URL and auth token for the 2nd part
let dataPart2 = Data(repeating: UInt8(ascii: "B"), count: (100 * 1024 * 1024))
let dataPart2ShaHexStr = ""

// Upload Part 2
let uploadUrlForPart2 = URL(string: "<upload part url obtained from b2_get_upload_part_url>")!
let uploadAuthTokenForPart2 = "<upload part authorization token obtained from b2_get_upload_part_url>"
var request = URLRequest(url: uploadUrlForPart2)
request.httpMethod = "POST"
request.addValue(uploadAuthTokenForPart2, forHTTPHeaderField: "Authorization")
request.addValue(String(2), forHTTPHeaderField: "X-Bz-Part-Number")
request.addValue(String(dataPart2.count), forHTTPHeaderField: "Content-Length")
request.addValue(dataPart2ShaHexStr, forHTTPHeaderField: "X-Bz-Content-Sha1")

let taskForPart2 = URLSession.shared.uploadTask(with: request, from: dataPart2) { (data, response, error) in
	if let jsonData = data {
		let json = String(data: jsonData, encoding: .utf8)
		print("\(json!)")
	}
}

// Start the task
taskForPart2.resume()

// Swift 4.1 (swiftlang-902.0.48 clang-902.0.37.1) Xcode 9.3.1 (9E501)

Output

Assuming ~200 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Code

require 'json'
require 'net/http'
require 'digest/sha1'

minimum_part_size_bytes = 100 * (1000 * 1000) # Obtained from b2_authorize_account. The default is 100 MB
upload_url = "" # Provided by b2_get_upload_url
local_file = "<Path to File to Upload>" # File to be uploaded
upload_authorization_token = "" # Provided by b2_get_upload_url
content_type = ""  # The content type of the file
local_file_path = "" # Path to the file you want to upload
local_file_size = File.stat(local_file_path).size 
total_bytes_sent = 0
bytes_sent_for_part = minimum_part_size_bytes
sha1_of_parts = Array.new # SHA1 of each uploaded part.  You will need to save these because you will need them in b2_finish_large_file
part_no = 1
while total_bytes_sent < local_file_size do
	
	# Determine num bytes to send	
	if ((local_file_size - total_bytes_sent) < minimum_part_size_bytes) 
		bytes_sent_for_part = (local_file_size - total_bytes_sent)
	end
	
	# Read file into memory and calculate a SHA1
	file_part_data = File.read(local_file_path, bytes_sent_for_part, total_bytes_sent, mode: "rb")
	sha1_of_parts.push(Digest::SHA1.hexdigest(file_part_data))
	hex_digest_of_part = sha1_of_parts[part_no - 1]
	
	# Send it over the wire
	uri = URI(upload_url)
	req = Net::HTTP::Post.new(uri)
	req.add_field("Authorization","#{upload_authorization_token}")
	req.add_field("X-Bz-Part-Number","#{part_no}")
	req.add_field("X-Bz-Content-Sha1","#{hex_digest_of_part}")
	req.add_field("Content-Length", "#{bytes_sent_for_part}")
	req.body = file_part_data
	http = Net::HTTP.new(req.uri.host, req.uri.port)
	http.use_ssl = (req.uri.scheme == 'https')
	res = http.start {|http| http.request(req)}
	case res
	when Net::HTTPSuccess then
	    json = res.body
	    puts json
	when Net::HTTPRedirection then
	    fetch(res['location'], limit - 1)
	else
	    res.error!
	end

	# Prepare for the next iteration of the loop
	total_bytes_sent += bytes_sent_for_part	
	#offset = total_bytes_sent
	part_no += 1
end

Output

Assuming ~208 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Part 3:
{
  "contentLength": 8158542,
  "contentSha1": "4546018a346df683acc9a3367977de4cc8024965",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 3
}

Code

using System;
using System.Net;
using System.Text;
using System.IO;
using System.Collections;
using System.Collections.Generic;
using System.Security.Cryptography;

FileInfo fileInfo = new FileInfo("<Path to file>");
long localFileSize = fileInfo.Length;
long totalBytesSent = 0;
long bytesSentForPart = 100 * (1000 * 1000);
long minimumPartSize = bytesSentForPart;
ArrayList partSha1Array = new ArrayList();
byte[] data = new byte[100 * (1000 * 1000)];
int partNo = 1;
while (totalBytesSent < localFileSize)
{
    if ((localFileSize - totalBytesSent) < minimumPartSize)
    {
        bytesSentForPart = (localFileSize - totalBytesSent);
    }

    // Generate SHA1
    FileStream f = File.OpenRead("<Path to file>");
    f.Seek(totalBytesSent, SeekOrigin.Begin);
    f.Read(data, 0, (int)bytesSentForPart);
    SHA1 sha1 = SHA1.Create();
    byte[] hashData = sha1.ComputeHash(data, 0, (int)bytesSentForPart);
    StringBuilder sb = new StringBuilder();
    foreach (byte b in hashData)
    {
        sb.Append(b.ToString("x2"));
    }
    f.Close();
    partSha1Array.Add(sb.ToString());

    HttpWebRequest uploadPartRequest = (HttpWebRequest)WebRequest.Create(getUploadUrlData.uploadUrl);
    uploadPartRequest.Method = "POST";
    uploadPartRequest.Headers.Add("Authorization", getUploadUrlData.authorizationToken);
    uploadPartRequest.Headers.Add("X-Bz-Part-Number", partNo.ToString());
    uploadPartRequest.Headers.Add("X-Bz-Content-Sha1", (String)partSha1Array[(partNo - 1)]);
    uploadPartRequest.ContentType = "application/json; charset=utf-8";
    uploadPartRequest.ContentLength = bytesSentForPart;
    using (Stream stream = uploadPartRequest.GetRequestStream())
    {
        stream.Write(data, 0, (int)bytesSentForPart);
        stream.Close();
    }
    HttpWebResponse uploadPartResponse = null;
    try
    {
        uploadPartResponse = (HttpWebResponse)uploadPartRequest.GetResponse();
    }
    catch (WebException e)
    {
        using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response)
        {
            Console.WriteLine("Error code: {0}", errorResponse.StatusCode);
            using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream()))
            {
                String text = reader.ReadToEnd();
                Console.WriteLine(text);
            }
        }
    }
    finally
    {
        uploadPartResponse.Close();
    }

    partNo++;
    totalBytesSent = totalBytesSent + bytesSentForPart;
}

Output

under construction

Code

<?php

// Used by curl when CURLOPT_READFUNCTION is set
function myReadFile($curl_rsrc, $file_pointer, $length) {
	return fread($file_pointer, $length);
}

// Upload parts
$minimum_part_size = 100 * (1000 * 1000); // Obtained from b2_authorize_account. The default is 100 MB
$local_file = "<path to large file>";
$local_file_size = filesize($local_file);
$total_bytes_sent = 0;
$bytes_sent_for_part = 0;
$bytes_sent_for_part = $minimum_part_size;
$sha1_of_parts = Array();
$part_no = 1;
$file_handle = fopen($local_file, "r");
while($total_bytes_sent < $local_file_size) {

	// Determine the number of bytes to send based on the minimum part size	
	if (($local_file_size - $total_bytes_sent) < $minimum_part_size) {
		$bytes_sent_for_part = ($local_file_size - $total_bytes_sent);
	}

	// Get a sha1 of the part we are going to send	
	fseek($file_handle, $total_bytes_sent);
	$data_part = fread($file_handle, $bytes_sent_for_part);
	array_push($sha1_of_parts, sha1($data_part));
	fseek($file_handle, $total_bytes_sent);

	// Send it over th wire
	$session = curl_init($upload_url);
	// Add headers
	$headers = array();
	$headers[] = "Accept: application/json";
	$headers[] = "Authorization: " . $large_file_auth_token;
	$headers[] = "Content-Length: " . $bytes_sent_for_part;
	$headers[] = "X-Bz-Part-Number: " . $part_no;
	$headers[] = "X-Bz-Content-Sha1: " . $sha1_of_parts[$part_no - 1];
	curl_setopt($session, CURLOPT_POST, true);
	curl_setopt($session, CURLOPT_HTTPHEADER, $headers);  // Add headers
	curl_setopt($session, CURLOPT_INFILE, $file_handle);
	curl_setopt($session, CURLOPT_INFILESIZE, (int)$bytes_sent_for_part);
	curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
	curl_setopt($session, CURLOPT_READFUNCTION, "myReadFile");
	$server_output = curl_exec($session);
	curl_close ($session);
	print $server_output . "\n";	

	// Prepare for the next iteration of the loop
	$part_no++;
	$total_bytes_sent = $bytes_sent_for_part + $total_bytes_sent;
	$read_file_bytes_read = 0;
}
fclose($file_handle);

?>

Output

Assuming ~208 MB File

Part 1:
{
  "contentLength": 100000000,
  "contentSha1": "062685a84ab248d2488f02f6b01b948de2514ad8",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 1
}

Part 2:
{
  "contentLength": 100000000,
  "contentSha1": "cf634751c3d9f6a15344f23cbf13f3fc9542addf",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 2
}

Part 3:
{
  "contentLength": 8158542,
  "contentSha1": "4546018a346df683acc9a3367977de4cc8024965",
  "fileId": "4_ze73ede9c9c8412db49f60715_f100b4e93fbae6252_d20150824_m224353_c900_v8881000_t0001",
  "partNumber": 3
}