Child Item

unreliable data transfer w/ API

Hi,

I'm using API to transfer data from my computer to Dropbox. For 50 500MB files 5 have failed with the error like this:

Uploading "/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.data" to "/bigWrkA/oneA.e15.oswrk136_500GB.data"..*** FAILED
An error occurred requesting /chunked_upload

Can you advice me how to overcome this issue?

Thanks

Jan

Find more posts tagged with

API

Abuse

Comments

Greg-DB

What's the actual error you're getting when making the failed calls? What chunk size are you using?

Jan B.23

What’s the actual error you're getting when making the failed calls?

The stderr reports for last 3 chunks of this file:

+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394232
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394232&oauth_nonce=14453'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=0+1
+ [[ 1 -gt 2 ]]
+ [[ 209715200 != 524288000 ]]
+ let OFFSET_MB=209715200/1024/1024
+ dd if=/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.data of=/tmp/du_chunk_1035 bs=1048576 skip=200 count=100
+ [[ 209715200 != 0 ]]
+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394296
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394296&oauth_nonce=5737'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
++ sed -n -e 's/{"error": "\([^"]*\)"}/\1/p' /tmp/du_resp_debug
+ ERROR_MSG=
+ case $ERROR_MSG in
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=1+1
+ [[ 2 -gt 2 ]]
+ [[ 209715200 != 524288000 ]]
+ let OFFSET_MB=209715200/1024/1024
+ dd if=/var/lib/condor/execute/dir_2220/oneA.e15.oswrk136_500GB.data of=/tmp/du_chunk_1035 bs=1048576 skip=200 count=100
+ [[ 209715200 != 0 ]]
+ CHUNK_PARAMS='upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200'
++ utime
+++ date +%s
++ echo 1434394336
+ curl -s --show-error --globoff -i -o /tmp/du_resp_debug --upload-file /tmp/du_chunk_1035 'https://api-content.dropbox.com/1/chunked_upload?upload_id=0B3fybsi14V1MmpZ18As_Q&offset=209715200&oauth_consumer_key=xeg3vxakd86svsh&oauth_token=jm624faonumhhxl9&oauth_signature_method=PLAINTEXT&oauth_signature=<redacted>%<redacted>&oauth_timestamp=1434394336&oauth_nonce=21291'
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
++ sed -n -e 's/{"error": "\([^"]*\)"}/\1/p' /tmp/du_resp_debug
+ ERROR_MSG=
+ case $ERROR_MSG in
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
+ let UPLOAD_ERROR=2+1
+ [[ 3 -gt 2 ]]
+ print ' FAILED\n'
+ [[ 0 == 0 ]]
+ echo -ne ' FAILED\n'
+ print 'An error occurred requesting /chunked_upload\n'
+ [[ 0 == 0 ]]
+ echo -ne 'An error occurred requesting /chunked_upload\n'
+ ERROR_STATUS=1
+ return
+ (( i++  ))
+ (( i<6 ))
+ remove_temp_files
+ [[ 1 == 0 ]]
+ exit 1

====================================================================================

What chunk size are you using?

CHUNK_SIZE=100

It is in MB. The code sending data looks like this:

#Chunked file upload
#$1 = Local source file
#$2 = Remote destination file
function db_chunked_upload_file
{
    local FILE_SRC=$(normalize_path "$1")
    local FILE_DST=$(normalize_path "$2")

    print " > Uploading \"$FILE_SRC\" to \"$FILE_DST\"

    local FILE_SIZE=$(file_size "$FILE_SRC")
    local OFFSET=0
    local UPLOAD_ID="
    local UPLOAD_ERROR=0

    local CHUNK_PARAMS="

    #Uploading chunks...
    while ([[ $OFFSET != $FILE_SIZE ]]); do

        let OFFSET_MB=$OFFSET/1024/1024

        #Create the chunk
        dd if="$FILE_SRC" of="$CHUNK_FILE" bs=1048576 skip=$OFFSET_MB count=$CHUNK_SIZE 2> /dev/null

        #Only for the first request these parameters are not included
        if [[ $OFFSET != 0 ]]; then
            CHUNK_PARAMS="upload_id=$UPLOAD_ID&offset=$OFFSET"
        fi

        #Uploading the chunk...
        $CURL_BIN $CURL_ACCEPT_CERTIFICATES -s --show-error --globoff -i -o "$RESPONSE_FILE" --upload-file "$CHUNK_FILE" "$API_CHUNKED_UPLOAD_URL?$CHUNK_PARAMS&oauth_consumer_key=$APPKEY&oauth_token=$OAUTH_ACCESS_TOKEN&oauth_signature_method=PLAINTEXT&oauth_signature=$APPSECRET%26$OAUTH_ACCESS_TOKEN_SECRET&oauth_timestamp=$(utime)&oauth_nonce=$RANDOM" 2> /dev/null
        check_http_response

        #Check
        if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
            print "."
            UPLOAD_ERROR=0
            UPLOAD_ID=$(sed -n 's/.*"upload_id": *"*\([^"]*\)"*.*/\1/p' "$RESPONSE_FILE")
            OFFSET=$(sed -n 's/.*"offset": *\([^}]*\).*/\1/p' "$RESPONSE_FILE")
        else
            print "*"
            let UPLOAD_ERROR=$UPLOAD_ERROR+1

            #On error, the upload is retried for max 3 times
            if [[ $UPLOAD_ERROR -gt 2 ]]; then
                print " FAILED\n"
                print "An error occurred requesting /chunked_upload\n"
                ERROR_STATUS=1
                return
            fi
        fi

    done

    UPLOAD_ERROR=0

    #Commit the upload
    while (true); do

        $CURL_BIN $CURL_ACCEPT_CERTIFICATES -s --show-error --globoff -i -o "$RESPONSE_FILE" --data "upload_id=$UPLOAD_ID&oauth_consumer_key=$APPKEY&oauth_token=$OAUTH_ACCESS_TOKEN&oauth_signature_method=PLAINTEXT&oauth_signature=$APPSECRET%26$OAUTH_ACCESS_TOKEN_SECRET&oauth_timestamp=$(utime)&oauth_nonce=$RANDOM" "$API_CHUNKED_UPLOAD_COMMIT_URL/$ACCESS_LEVEL/$(urlencode "$FILE_DST")" 2> /dev/null
        check_http_response

        #Check
        if grep -q "^HTTP/1.1 200 OK" "$RESPONSE_FILE"; then
            print "."
            UPLOAD_ERROR=0
            break
        else
            print "*"
            let UPLOAD_ERROR=$UPLOAD_ERROR+1

            #On error, the commit is retried for max 3 times
            if [[ $UPLOAD_ERROR -gt 2 ]]; then
                print " FAILED\n"
                print "An error occurred requesting /commit_chunked_upload\n"
                ERROR_STATUS=1
                return
            fi
        fi

    done

    print " DONE\n"
}

Greg-DB

Thanks for the additional information. I don't think I see the actual response or error in there though. If you get a response from the API, it should contain an error message in the body. If you don't (e.g., if the connection fails) I would expect curl to show a network error of some sort. Can you print those out?

In any case, 100 MB is a pretty large chunk size, so I recommend trying something smaller, perhaps 50 MB or smaller. (API calls for large chunks can time out.) Let me know if that doesn't help.