3

I used curl with char URL[]= "file:///d:/temporal/prueba1.txt" in order to test using a hard drive file in order to tet received data.

I have following problems:

  1. Skips first 2 chars in large files: -solved- it was an error reading the data
  2. I would like read only one time (preferally first 2000 characters) It exist a curl command to do it?
  3. If I make at writer file: long longitud=strlen(datain), I obtain longitud much larger than size*nmemb
  4. Curl can't go through corporate network, but URLOpenBlockingStream() + read() can do it
  5. Sometimes Curl readback a lot of erroneus data from url. I think that a connection error happened but I does not understand

This is the program. (Note: pull_one_url() is called by using threads):

static void *pull_one_url(void *ii){
ix_bloque=initialize();
     lee_curl_c *lee_curl;//Curl class
CURL *curl; 
CURLcode result; 
curl = curl_easy_init(); // Create our curl handle 
char errorBuffer[CURL_ERROR_SIZE]=""; 
char user_agent[]="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
// Write all expected data in here 

if (curl) 
{ 
    curl_easy_setopt(curl, CURLOPT_USERAGENT,user_agent);
    curl_easy_setopt(curl, CURLOPT_PORT, PC_PORT_a); // Check this before using 
    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorBuffer); 
    curl_easy_setopt(curl, CURLOPT_URL, lee_curl->url[ix_bloque]); 
    curl_easy_setopt(curl, CURLOPT_HEADER, 0); 
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, D_TIMEOUT);//240 segundos
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);//SOLO primer http, ignorar sucesivos
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &writer); 
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, lee_curl->buffer[ix_bloque]); 
    curl_easy_setopt(curl, CURLOPT_VERBOSE, VERBOSE_url);//Para ver que esta pasando
    // Attempt to retrieve the remote page  
    result = curl_easy_perform(curl); 
    if (errorBuffer[0])
    {
        fprintf(stderr, "\n##Tarea %li ERROR EN    %s DEBIDO A: %s",ix_bloque,lee_curl->url[ix_bloque],errorBuffer);
        lee_curl->estado[ix_bloque]=2;//error
    }
    else
    {
        fprintf(stderr, "\n##Tarea %li NO ERROR EN %s          ",ix_bloque,lee_curl->url[ix_bloque]);
        lee_curl->estado[ix_bloque]=1;//Lleno
    }
    // Always cleanup 
    curl_easy_cleanup(curl); 
} 
return NULL;    }

Here is the writer function:

static size_t writer(void *vdata, size_t size, size_t nmemb, void *vbuffer_in){ 
char *datain=(char *) vdata;
char *buffer_in=(char *) vbuffer_in;
nmemb*=size;//No multiplication needed anymore
//long longitud=strlen(datain);//longitud always larger than nmemb 
// Is there anything in the buffer? 
if ( (buffer_in != NULL) && (buffer_in[0]=='\0') ) //CAUTION, reads only if buffer is empty!!!
{ 
    //strcat(buffer_in,data); //SIRVE si el fichero es de mas de 16kb OJO!
    if (nmemb<(D_BUFFER_SIZE-1))
    {
        memcpy(buffer_in,datain,nmemb);
        buffer_in[nmemb]='\0';//end of string
        fprintf(stderr,"\n###SUCCESSFUL, loaded :      %li bytes",nmemb);
    }
    else
    {
        memcpy(buffer_in,data,D_BUFFER_SIZE);
        buffer_in[D_BUFFER_SIZE-1]='\0';//end of string
        fprintf(stderr,"\n###WARNING, trying to load   %li LARGER THAN %li",nmemb,D_BUFFER_SIZE);
        fprintf(stderr," El tamano maximo curl=%li = %likB",CURL_MAX_WRITE_SIZE,CURL_MAX_WRITE_SIZE/1024);
    }
return nmemb; 
} 
return 0;     }

1 Answer 1

2

CURLOPT_RANGE allows you to request a part of a file/resource:

http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTRANGE

2
  • Thank you. I discovered that curl changes ASCII: "0D 0A" to "0D 0d 0A"
    – Pleyades
    Feb 11, 2011 at 8:35
  • OK!! you are right Daniel. The problem if when writting at file using fprintf that adds a 0x0d before carriage return. So I had to open in binary mode: fopen("output.txt","wb") @Daniel Stenberg
    – Pleyades
    Feb 15, 2011 at 10:01

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.