LinuxQuestions.org
Visit Jeremy's Blog.
Go Back   LinuxQuestions.org > Forums > Non-*NIX Forums > Programming
User Name
Password
Programming This forum is for all programming questions.
The question does not have to be directly related to Linux and any language is fair game.

Notices


Reply
  Search this Thread
Old 11-11-2022, 03:01 PM   #1
gda
Member
 
Registered: Oct 2015
Posts: 130

Rep: Reputation: 26
Getting HTTP error code 404 using a simple C https client


Hi all,

quite strange problem here... I have developed a simple https client to get some data from an http server (nginx) running a public API (no authentication is needed). The problem is that when I try to get the data I always got a 404 HTTP error even if I'm sure the path of the resource I'm requesting is correct in the GET command I sent to the server. This is also confirmed by the fact that if I try to get the same resource (using exactly the same path) with curl it works without any problem.

This is the https client I wrote (following an example I found on the web):

Code:
int main(int argc, char **argv)
{

  char *message=NULL;

  char *CAfile=NULL;
  int port=443;
  
  char *host_and_port=NULL;
  
  unsigned int len=0;
  
  SSL *ssl=NULL;
  BIO *bio=NULL; 
  SSL_CTX *ctx=NULL;

  
  int r = 0;
  ssize_t rr = -1;
  ssize_t length=0;

  unsigned char *buffer=NULL;
  ssize_t l=0;
 
  unsigned char *https_stream=NULL;
  ssize_t https_stream_size=0;
 
  CAfile=strdup("mycert.pem");

  message=strdup("GET /path1/path2?arg1=val1&arg2=val2 HTTP/1.0\r\nHost: myhost.com\r\n\r\n");
  host_and_port=strdup("myhost.com:443");  
  
  SSL_load_error_strings();
  SSL_library_init();
  ERR_load_BIO_strings();
  OpenSSL_add_all_algorithms();

  if (!(ctx = SSL_CTX_new(TLS_client_method())))
    {
      fprintf(stderr,"The creation of a new SSL_CTX object failed.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  
  
  if (!(r = SSL_CTX_load_verify_locations(ctx, CAfile, NULL)))
    {
      fprintf(stderr,"Unable to load the trust certificate from file: %s\n",CAfile);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  
  /* Setting up the BIO SSL object */
  bio = BIO_new_ssl_connect(ctx);
  BIO_get_ssl(bio, &ssl);
  if (!ssl)
    {
      fprintf(stderr,"Unable to allocate SSL pointer.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
  
  /* Attempt to connect */
  BIO_set_conn_hostname(bio, host_and_port);

  /* Verify the connection opened and perform the handshake */
  if (BIO_do_connect(bio) < 1)
    {
      fprintf(stderr,"Unable to connect BIO. %s\n",host_and_port);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));   
      return -1;
    }

  length=strlen(message);
 
  rr=-1;
  while (rr < 0)
    {
      rr = BIO_write(bio, message, length);
      if (rr <= 0)
    {
      if (!BIO_should_retry(bio))
        {
          fprintf(stderr,"BIO_write should retry.\n");
          fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
          fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
          return -1;
        }
    }
    }
  
  if (rr!=length)
    {
      fprintf(stderr,"Error in sending encripted message.\n");
      return -1;
    }

  if (buffer)
    {
      free(buffer);
      buffer=NULL;
    }
  
  
  if (!(https_stream=malloc(sizeof(char))))
    {
      fprintf(stderr, "Allocation memory failed (https_stream string), code=%d (%s)\n",errno, strerror(errno));
      return LIBCLOUDNET_MEMORY_ERROR;
    }
  https_stream[0]=0;

  https_stream_size=0;
  length=4096;
  if (!(buffer=malloc(length*sizeof(char))))
    {
      fprintf(stderr, "Allocation memory failed (buffer string), code=%d (%s)\n",errno, strerror(errno));
      return -1;
    }
  
  bzero(buffer,length);   

  rr=-1;
  while((rr = BIO_read(bio, buffer, length)))
    {
      if (rr<0)
    if (!BIO_should_retry(bio))
      {
        fprintf(stderr,"BIO_read should retry.\n");
        fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
        fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
        return -1;
      }
      l=https_stream_size;
      https_stream_size += rr;
      if (!(https_stream=realloc(https_stream,https_stream_size*sizeof(char))))
        {
          fprintf(stderr, "Re-allocation memory failed (*https_stream string), code=%d (%s)\n",errno, strerror(errno));
          return -1;
    }
      memcpy(https_stream+l,buffer,rr);
      bzero(buffer,length);
    }
  
  /* clean up the SSL context resources for the encrypted link */
  SSL_CTX_free(ctx);

  
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"HTTP Request\n");
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"%s\n",message);
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"\n");
  fprintf(stdout,"Host and port: %s\n",host_and_port);
  fprintf(stdout,"\n");
  
  fprintf(stdout,"HTTP Response\n");
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"%s\n",https_stream);
  fprintf(stdout,"==========================================\n");
  
  
  
  if (message)
    {
      free(message);
      message=NULL;
    }

  if (buffer)
    {
      free(buffer);
      buffer=NULL;
    }

  
  if (https_stream)
    {
      free(https_stream);
      https_stream=NULL;
    }
  
  if (host_and_port)
    {
      free(host_and_port);
      host_and_port=NULL;
    }
 
  
  return 0;
}
where "myhost.com" and "/path1/path2?arg1=val1&arg2=val2" represent the complete address of the host and the path of the resource I want to get respectively.
This code compiles Just fine (without any warning) and the output is:

Code:
==========================================
HTTP Request
==========================================
GET /path1/path2?arg1=val1&arg2=val2 HTTP/1.0
Host: myhost.com


==========================================

Host and port: myhost.com:443

HTTP Response
==========================================
HTTP/1.1 404 Not Found
Content-Length: 231
x-amz-request-id: tx00000000000000019debd-00636e7b40-39e6dd1-default
Accept-Ranges: bytes
Content-Type: application/xml
Date: Fri, 11 Nov 2022 16:41:36 GMT
X-Frame-Options: DENY
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
Content-Security-Policy:script-src: https://www.google-analytics.com https://q.quora.com
Referrer-Policy: no-referrer-when-downgrade
Strict-Transport-Security: max-age=31536000;includeSubDomains;preload

<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchBucket</Code><BucketName>myhost.com</BucketName><RequestId>tx00000000000000019debd-00636e7b40-39e6dd1-default</RequestId><HostId>39e6dd1-default-default</HostId></Error>
==========================================
So, as you can see, I got HTTP error code 404.

Now if from the same machine I use curl to access to the same resource on the same server I got:

Code:
curl -v --http1.0 "https://myhost.com/path1/path2?arg1=val1&arg2=val2"
*   Trying <host_IP>:443...
* Connected to myhost.com (<host_IP>) port 443 (#0)
* ALPN: offers http/1.1
* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:@STRENGTH
*  CAfile: none
*  CApath: /etc/ssl/certs
* TLSv1.2 (OUT), TLS header, Certificate Status (22):
* TLSv1.2 (OUT), TLS handshake, Client hello (1):
* TLSv1.2 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Change cipher spec (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS change cipher, Change cipher spec (1):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES256-GCM-SHA384
* ALPN: server did not agree on a protocol. Uses default.
* Server certificate:
...
*  SSL certificate verify ok.
> GET /path1/path2?arg1=val1&arg2=val2 HTTP/1.0
> Host: myhost.com
> User-Agent: curl/7.85.0
> Accept: */*
> 
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< server: nginx/1.23.2
< date: Fri, 11 Nov 2022 17:40:16 GMT
< content-type: application/json; charset=utf-8
< transfer-encoding: chunked
< x-powered-by: Express
< set-cookie: 3866fdcd36aeaae468ebac902177effe=5f9cbf1450843f212673d67cb86a2f9a; path=/; HttpOnly
< cache-control: private
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-Xss-Protection: 1; mode=block
< Content-Security-Policy:script-src: https://www.google-analytics.com https://q.quora.com
< Referrer-Policy: no-referrer-when-downgrade
< Strict-Transport-Security: max-age=31536000;includeSubDomains;preload
<

As you can see curl works fine returning HTTP code 200. This proves that on the server the resource I was looking for is actually available at the specified path (which is the same I have specified in both curl and my client). Consequently there should be something wrong in the client I wrote which unfortunately I cannot figure out. Any idea on what could be? Any help is really welcome.

Unfortunately I do not have access to the server log files....

Thanks in advance for your availability!
 
Old 11-11-2022, 04:13 PM   #2
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
You didn't tell what your includes are. I guessed.
Code:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <errno.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
Code:
gcc -I/usr/include/openssl/ test.c -o test
test.c: In function ‘main’:
test.c:110:16: error: ‘LIBCLOUDNET_MEMORY_ERROR’ undeclared (first use in this function)
  110 |         return LIBCLOUDNET_MEMORY_ERROR;
      |                ^~~~~~~~~~~~~~~~~~~~~~~~
test.c:110:16: note: each undeclared identifier is reported only once for each function it appears in
You want to give some more info?
 
Old 11-11-2022, 05:02 PM   #3
gda
Member
 
Registered: Oct 2015
Posts: 130

Original Poster
Rep: Reputation: 26
You are right I forgot the report the includes. Sorry for that. Here is the complete code:

Code:
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <errno.h>
#include <math.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

int main(int argc, char **argv)
{

  char *message=NULL;

  char *CAfile=NULL;
  int port=443;
  
  char *host_and_port=NULL;
  
  unsigned int len=0;
  
  SSL *ssl=NULL;
  BIO *bio=NULL; 
  SSL_CTX *ctx=NULL;

  
  int r = 0;
  ssize_t rr = -1;
  ssize_t length=0;

  unsigned char *buffer=NULL;
  ssize_t l=0;
 
  unsigned char *https_stream=NULL;
  ssize_t https_stream_size=0;
 
  CAfile=strdup("mycert.pem");

  message=strdup("GET /path1/path2?arg1=val1&arg2=val2 HTTP/1.0\r\nHost: myhost.com\r\n\r\n");
  host_and_port=strdup("myhost.com:443");  
  
  SSL_load_error_strings();
  SSL_library_init();
  ERR_load_BIO_strings();
  OpenSSL_add_all_algorithms();

  if (!(ctx = SSL_CTX_new(TLS_client_method())))
    {
      fprintf(stderr,"The creation of a new SSL_CTX object failed.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  
  
  if (!(r = SSL_CTX_load_verify_locations(ctx, CAfile, NULL)))
    {
      fprintf(stderr,"Unable to load the trust certificate from file: %s\n",CAfile);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  
  /* Setting up the BIO SSL object */
  bio = BIO_new_ssl_connect(ctx);
  BIO_get_ssl(bio, &ssl);
  if (!ssl)
    {
      fprintf(stderr,"Unable to allocate SSL pointer.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
  SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
  
  /* Attempt to connect */
  BIO_set_conn_hostname(bio, host_and_port);

  /* Verify the connection opened and perform the handshake */
  if (BIO_do_connect(bio) < 1)
    {
      fprintf(stderr,"Unable to connect BIO. %s\n",host_and_port);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));   
      return -1;
    }

  length=strlen(message);
 
  rr=-1;
  while (rr < 0)
    {
      rr = BIO_write(bio, message, length);
      if (rr <= 0)
    {
      if (!BIO_should_retry(bio))
        {
          fprintf(stderr,"BIO_write should retry.\n");
          fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
          fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
          return -1;
        }
    }
    }
  
  if (rr!=length)
    {
      fprintf(stderr,"Error in sending encripted message.\n");
      return -1;
    }

  if (buffer)
    {
      free(buffer);
      buffer=NULL;
    }
  
  
  if (!(https_stream=malloc(sizeof(char))))
    {
      fprintf(stderr, "Allocation memory failed (https_stream string), code=%d (%s)\n",errno, strerror(errno));
      return -1;
    }
  https_stream[0]=0;

  https_stream_size=0;
  length=4096;
  if (!(buffer=malloc(length*sizeof(char))))
    {
      fprintf(stderr, "Allocation memory failed (buffer string), code=%d (%s)\n",errno, strerror(errno));
      return -1;
    }
  
  bzero(buffer,length);   

  rr=-1;
  while((rr = BIO_read(bio, buffer, length)))
    {
      if (rr<0)
    if (!BIO_should_retry(bio))
      {
        fprintf(stderr,"BIO_read should retry.\n");
        fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
        fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
        return -1;
      }
      l=https_stream_size;
      https_stream_size += rr;
      if (!(https_stream=realloc(https_stream,https_stream_size*sizeof(char))))
        {
          fprintf(stderr, "Re-allocation memory failed (*https_stream string), code=%d (%s)\n",errno, strerror(errno));
          return -1;
    }
      memcpy(https_stream+l,buffer,rr);
      bzero(buffer,length);
    }
  
  /* clean up the SSL context resources for the encrypted link */
  SSL_CTX_free(ctx);

  
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"HTTP Request\n");
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"%s\n",message);
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"\n");
  fprintf(stdout,"Host and port: %s\n",host_and_port);
  fprintf(stdout,"\n");
  
  fprintf(stdout,"HTTP Response\n");
  fprintf(stdout,"==========================================\n");
  fprintf(stdout,"%s\n",https_stream);
  fprintf(stdout,"==========================================\n");
  
  
  
  if (message)
    {
      free(message);
      message=NULL;
    }

  if (buffer)
    {
      free(buffer);
      buffer=NULL;
    }

  
  if (https_stream)
    {
      free(https_stream);
      https_stream=NULL;
    }
  
  if (host_and_port)
    {
      free(host_and_port);
      host_and_port=NULL;
    }
 
  
  return 0;
}
The code should compile as it follows (maybe you need to add -I and/or -L options if in your system the required includes and libs are not in standard paths):

Code:
gcc -o test -Wall -lssl -lcrypto test.c
 
Old 11-11-2022, 05:48 PM   #4
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
Code:
openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out mycert.pem
I think that your code is hard to read the way you posted it. I reformatted it, and changed the url.

This works.

test4.c
Code:
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <errno.h>
#include <math.h>
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

int main(int argc, char **argv)
{
    char *message=NULL;
    char *CAfile=NULL;
    int port=443;
    char *host_and_port=NULL;
    unsigned int len=0;

    SSL *ssl=NULL;
    BIO *bio=NULL; 
    SSL_CTX *ctx=NULL;

    int r = 0;
    ssize_t rr = -1;
    ssize_t length=0;

    unsigned char *buffer=NULL;
    ssize_t l=0;

    unsigned char *https_stream=NULL;
    ssize_t https_stream_size=0;

    CAfile=strdup("mycert.pem");

    //message=strdup("GET /path1/path2?arg1=val1&arg2=val2 HTTP/1.0\r\nHost: myhost.com\r\n\r\n");
    //host_and_port=strdup("myhost.com:443"); 
    
    message=strdup("GET https://www.linuxquestions.org/questions/lqsearch.php?do=getnew&daysprune=7 HTTP/1.0\r\nHost: myhost.com\r\n\r\n");
    host_and_port=strdup("linuxquestions.org:443"); 

    SSL_load_error_strings();
    SSL_library_init();
    ERR_load_BIO_strings();
    OpenSSL_add_all_algorithms();

    if (!(ctx = SSL_CTX_new(TLS_client_method()))) {
      fprintf(stderr,"The creation of a new SSL_CTX object failed.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }

    if (!(r = SSL_CTX_load_verify_locations(ctx, CAfile, NULL))) {
      fprintf(stderr,"Unable to load the trust certificate from file: %s\n",CAfile);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }

    /* Setting up the BIO SSL object */
    bio = BIO_new_ssl_connect(ctx);
    BIO_get_ssl(bio, &ssl);
    if (!ssl) {
      fprintf(stderr,"Unable to allocate SSL pointer.\n");
      fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));
      return -1;
    }
    SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);

    /* Attempt to connect */
    BIO_set_conn_hostname(bio, host_and_port);

    /* Verify the connection opened and perform the handshake */
    if (BIO_do_connect(bio) < 1) {
      fprintf(stderr,"Unable to connect BIO. %s\n",host_and_port);
      fprintf(stderr, "Error: %s\n",ERR_reason_error_string(ERR_get_error()));
      fprintf(stderr, "%s:",ERR_error_string(ERR_get_error(), NULL));   
      return -1;
    }

    length=strlen(message);

    rr=-1;
    while (rr < 0) {
        rr = BIO_write(bio, message, length);
        if (rr <= 0) {
            if (!BIO_should_retry(bio)) {
              fprintf(stderr,"BIO_write should retry.\n");
              fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
              fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
              return -1;
            }
        }
    }

    if (rr!=length) {
        fprintf(stderr,"Error in sending encripted message.\n");
        return -1;
    }

    if (buffer) {
        free(buffer);
        buffer=NULL;
    }


    if (!(https_stream=malloc(sizeof(char)))) {
        fprintf(stderr, 
            "Allocation memory failed (https_stream string), code=%d (%s)\n", 
            errno, strerror(errno));
        return -1;
    }
    
    https_stream[0]=0;
    https_stream_size=0;
    length=4096;
    
    if (!(buffer=malloc(length*sizeof(char)))) {
        fprintf(stderr, 
        "Allocation memory failed (buffer string), code=%d (%s)\n", 
        errno, strerror(errno));
        return -1;
    }

    bzero(buffer,length);   
    rr=-1;
    
    while((rr = BIO_read(bio, buffer, length))) {
        if (rr<0) {
            if (!BIO_should_retry(bio)) {
                fprintf(stderr,"BIO_read should retry.\n");
                fprintf(stderr, "Error: %s\n", ERR_reason_error_string(ERR_get_error()));
                fprintf(stderr, "%s:", ERR_error_string(ERR_get_error(), NULL));
                return -1;
            }
        }
        
        l=https_stream_size;
        https_stream_size += rr;
      
        if (!(https_stream=realloc(https_stream,https_stream_size*sizeof(char)))) {
          fprintf(stderr, 
          "Re-allocation memory failed (*https_stream string), code=%d (%s)\n", 
          errno, strerror(errno));
          return -1;
        }
    
        memcpy(https_stream+l,buffer,rr);
        bzero(buffer,length);
    }

    /* clean up the SSL context resources for the encrypted link */
    SSL_CTX_free(ctx);

    fprintf(stdout,"==========================================\n");
    fprintf(stdout,"HTTP Request\n");
    fprintf(stdout,"==========================================\n");
    fprintf(stdout,"%s\n",message);
    fprintf(stdout,"==========================================\n");
    fprintf(stdout,"\n");
    fprintf(stdout,"Host and port: %s\n",host_and_port);
    fprintf(stdout,"\n");

    fprintf(stdout,"HTTP Response\n");
    fprintf(stdout,"==========================================\n");
    fprintf(stdout,"%s\n",https_stream);
    fprintf(stdout,"==========================================\n");

    if (message) {
        free(message);
        message=NULL;
    }

    if (buffer) {
        free(buffer);
        buffer=NULL;
    }

    if (https_stream) {
        free(https_stream);
        https_stream=NULL;
    }

    if (host_and_port) {
        free(host_and_port);
        host_and_port=NULL;
    }
    return 0;
}
Code:
gcc test4.c -o test4 -Wall -lssl -lcrypto
Code:
./test4 | less
==========================================
HTTP Request
==========================================
GET https://www.linuxquestions.org/questions/lqsearch.php?do=getnew&daysprune=7 HTTP/1.0
Host: myhost.com


==========================================

Host and port: linuxquestions.org:443

HTTP Response
==========================================
HTTP/1.0 200 OK
Server: nginx
Date: Fri, 11 Nov 2022 22:46:10 GMT
Content-Type: text/html; charset=ISO-8859-1
Vary: Accept-Encoding
Set-Cookie: bblastvisit=1668206770; expires=Sat, 11-Nov-2023 22:46:10 GMT; path=/
Set-Cookie: bblastactivity=0; expires=Sat, 11-Nov-2023 22:46:10 GMT; path=/
Cache-Control: private
Pragma: private

Last edited by teckk; 11-11-2022 at 05:51 PM.
 
Old 11-11-2022, 07:55 PM   #5
sundialsvcs
LQ Guru
 
Registered: Feb 2004
Location: SE Tennessee, USA
Distribution: Gentoo, LFS
Posts: 10,009
Blog Entries: 4

Rep: Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607Reputation: 3607
Curious why you are using "C" for this ... with other tools this could almost be a "one-liner."
 
Old 11-12-2022, 09:29 AM   #6
gda
Member
 
Registered: Oct 2015
Posts: 130

Original Poster
Rep: Reputation: 26
Hi teckk,

first of all thanks a lot for your time efforts. This is very appreciated!

I know that in general my code works as your example in getting linuxquestions web page contents shows. Anyway the same code seems to fail in getting data from the specific server I was looking for. I don't think this is due to a misconfiguration of the web server because in that case also curl should fail in doing the same job and this is not the case. So I think something in my code is not handled correctly... but don't know what exactly...

I provide below an url where my code doesn't work but curl works well.

Using my code: starting from the test4.c you have provided and modifying the url as it follow:

Code:
message=strdup("GET https://cloudnet.fmi.fi/api/model-files?site=munich&date=2022-11-10 HTTP/1.0\r\nHost: cloudnet.fmi.fi\r\n\r\n");
host_and_port=strdup("cloudnet.fmi.fi:443");
I got the following output:

Code:
==========================================
HTTP Request
==========================================
GET https://cloudnet.fmi.fi/api/model-files?site=munich&date=2022-11-10 HTTP/1.0
Host: cloudnet.fmi.fi


==========================================

Host and port: cloudnet.fmi.fi:443

HTTP Response
==========================================
HTTP/1.1 404 Not Found
Content-Length: 231
x-amz-request-id: tx00000000000000019e6bd-00636fa8a4-39e6dd1-default
Accept-Ranges: bytes
Content-Type: application/xml
Date: Sat, 12 Nov 2022 14:07:32 GMT
Connection: close
X-Frame-Options: DENY
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
Content-Security-Policy:script-src: https://www.google-analytics.com https://q.quora.com
Referrer-Policy: no-referrer-when-downgrade
Strict-Transport-Security: max-age=31536000;includeSubDomains;preload

<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchBucket</Code><BucketName>cloudnet.fmi.fi</BucketName><RequestId>tx00000000000000019e6bd-00636fa8a4-39e6dd1-default</RequestId><HostId>39e6dd1-default-default</HostId></Error>
Using curl

Code:
curl -v --http1.0 'https://cloudnet.fmi.fi/api/model-files?site=munich&date=2022-11-10'
*   Trying 193.166.223.85:443...
* Connected to cloudnet.fmi.fi (193.166.223.85) port 443 (#0)
* ALPN: offers http/1.1
* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:@STRENGTH
*  CAfile: none
*  CApath: /etc/ssl/certs
* TLSv1.2 (OUT), TLS header, Certificate Status (22):
* TLSv1.2 (OUT), TLS handshake, Client hello (1):
* TLSv1.2 (IN), TLS handshake, Server hello (2):
* TLSv1.2 (IN), TLS handshake, Certificate (11):
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
* TLSv1.2 (IN), TLS handshake, Server finished (14):
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
* TLSv1.2 (OUT), TLS change cipher, Change cipher spec (1):
* TLSv1.2 (OUT), TLS handshake, Finished (20):
* TLSv1.2 (IN), TLS change cipher, Change cipher spec (1):
* TLSv1.2 (IN), TLS handshake, Finished (20):
* SSL connection using TLSv1.2 / ECDHE-RSA-AES256-GCM-SHA384
* ALPN: server did not agree on a protocol. Uses default.
* Server certificate:
*  subject: C=FI; ST=Uusimaa; O=Ilmatieteen laitos; CN=*.out.ocp.fmi.fi
*  start date: Mar 17 00:00:00 2022 GMT
*  expire date: Mar 17 23:59:59 2023 GMT
*  subjectAltName: host "cloudnet.fmi.fi" matched cert's "*.fmi.fi"
*  issuer: C=NL; O=GEANT Vereniging; CN=GEANT OV RSA CA 4
*  SSL certificate verify ok.
> GET /api/model-files?site=munich&date=2022-11-10 HTTP/1.0
> Host: cloudnet.fmi.fi
> User-Agent: curl/7.85.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< server: nginx/1.23.2
< date: Sat, 12 Nov 2022 14:09:15 GMT
< content-type: application/json; charset=utf-8
< x-powered-by: Express
< set-cookie: 3866fdcd36aeaae468ebac902177effe=5f9cbf1450843f212673d67cb86a2f9a; path=/; HttpOnly
< cache-control: private
< connection: close
< X-Frame-Options: DENY
< X-Content-Type-Options: nosniff
< X-Xss-Protection: 1; mode=block
< Content-Security-Policy:script-src: https://www.google-analytics.com https://q.quora.com
< Referrer-Policy: no-referrer-when-downgrade
< Strict-Transport-Security: max-age=31536000;includeSubDomains;preload
I also tried to sent the exact same message curl sents but with no luck:
Code:
message=strdup("GET /api/model-files?site=munich&date=2022-11-10 HTTP/1.0\r\nHost: cloudnet.fmi.fi\r\n\r\n");
What's wrong in what I'm doing?
 
Old 11-12-2022, 09:37 AM   #7
gda
Member
 
Registered: Oct 2015
Posts: 130

Original Poster
Rep: Reputation: 26
Quote:
Originally Posted by sundialsvcs View Post
Curious why you are using "C" for this ... with other tools this could almost be a "one-liner."
Because this code is a part of a much bigger program written in C where I would avoid to call external tools using the OS shell. So my first choice was to use ssl libraries to handle encrypted sockets. Do you know other easier options for doing similar job?
 
Old 11-12-2022, 10:23 AM   #8
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
Code:
#include <stdio.h>
#include <curl/curl.h>

int main(void)
{
    CURL *curl;
    FILE *fp;
    CURLcode res;
    char *url = "https://cloudnet.fmi.fi/api/model-files?site=munich&date=2022-11-10";
    char outfilename[FILENAME_MAX] = "page.html";
    curl = curl_easy_init();
                        
    if (curl) {   
        fp = fopen(outfilename,"wb");
        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
        res = curl_easy_perform(curl);
        curl_easy_cleanup(curl);
        fclose(fp);
    }   
    return 0;
}

//gcc test5.c -lcurl -o test5
Code:
cat page.html
[{"uuid":"b39a42b1-93c1-40e7-941d-167129231d42","version":"","pid":"","volatile":true,
"legacy":false,"quality":"nrt","measurementDate":"2022-11-10","checksum":"1ba002741bcf
21221862ed53fd09775365a8b54d2adaddbc872379b96e0c1abf","size":"561393","format":"HDF5
 (NetCDF4)","errorLevel":"pass","createdAt":"2022-11-11T07:48:14.229Z","updatedAt":
"2022-11-11T21:35:46.126Z","processingVersion":"2.8.0","modelId":"ecmwf","site":
{"id":"munich","humanReadableName":"Munich"
...

Last edited by teckk; 11-12-2022 at 10:25 AM.
 
1 members found this post helpful.
Old 11-12-2022, 11:14 AM   #9
gda
Member
 
Registered: Oct 2015
Posts: 130

Original Poster
Rep: Reputation: 26
Thanks again teckk! This really helps! I will give a try to libcurl as well.

Anyway do you have an idea why my original code does not work with this URL? For practical reasons I would prefer to patch (if possibile) my existing code instead of changing libs completely.

Anyway I have to admit that libcurl offers much easier implementation. I wonder why I didn't go for it since the beginning.
 
Old 11-12-2022, 12:29 PM   #10
boughtonp
Senior Member
 
Registered: Feb 2007
Location: UK
Distribution: Debian
Posts: 2,983

Rep: Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120Reputation: 2120
Quote:
Originally Posted by gda View Post
For practical reasons I would prefer to patch (if possibile) my existing code instead of changing libs completely.
You patch this bug and you'll find another next week, and another after that, and so on.

Or you could go with the reliable, widely-used and well-tested library written by experts.

 
Old 11-12-2022, 01:59 PM   #11
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
I was playing with this a little more because I was interested.

test8.c
Code:
#include <stdio.h> 
#include <memory.h> 
#include <errno.h> 
#include <sys/types.h> 
#include <sys/socket.h> 
#include <netinet/in.h> 
#include <arpa/inet.h> 
#include <netdb.h> 

#include <openssl/crypto.h> 
#include <openssl/x509.h> 
#include <openssl/pem.h> 
#include <openssl/ssl.h> 
#include <openssl/err.h>
 
void main(int argc, char **argv) 
{ 
    SSL *ssl; 
    SSL_CTX *ctx; 
    SSL_METHOD *client_method; 
    X509 *server_cert; 
    int sd,err; 
    char *str,*hostname,outbuf[4096],inbuf[4096],host_header[512]; 
    struct hostent *host_entry; 
    struct sockaddr_in server_socket_address; 
    struct in_addr ip; 

    SSLeay_add_ssl_algorithms( ); 
    client_method = SSLv23_client_method( ); 
    SSL_load_error_strings( ); 
    ctx = SSL_CTX_new(client_method); 

    printf("(1) SSL context initialized\n\n"); 

    hostname = argv[1]; 
    host_entry = gethostbyname(hostname); 
    bcopy(host_entry->h_addr, &(ip.s_addr), host_entry->h_length); 

    printf("(2) '%s' has IP address '%s'\n\n", hostname, inet_ntoa(ip)); 

    sd = socket (AF_INET, SOCK_STREAM, 0); 

    memset(&server_socket_address, ' ', sizeof(server_socket_address));
    memset(&server_socket_address, '\0', sizeof(server_socket_address)); 

    server_socket_address.sin_family = AF_INET; 
    server_socket_address.sin_port = htons(443); 
    memcpy(&(server_socket_address.sin_addr.s_addr), 
    host_entry->h_addr, host_entry->h_length); 

    err = connect(sd, (struct sockaddr*) &server_socket_address, 
    sizeof(server_socket_address)); 
    if (err < 0) { perror("can't connect to server port"); exit(1); } 

    printf("(3) TCP connection open to host '%s', port %d\n\n", 
    hostname, server_socket_address.sin_port); 

    ssl = SSL_new(ctx);
    SSL_set_fd(ssl, sd); 
    err = SSL_connect(ssl); 

    printf("(4) SSL endpoint created & handshake completed\n\n");  
    printf("(5) SSL connected with cipher: %s\n\n", SSL_get_cipher(ssl)); 

    server_cert = SSL_get_peer_certificate(ssl); 

    printf("(6) server's certificate was received:\n\n"); 

    str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0); 
    printf(" subject: %s\n", str); 

    str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0); 
    printf(" issuer: %s\n\n", str); 

    X509_free(server_cert); 

    sprintf(host_header,"Host: %s:443\r\n",hostname); 
    strcpy(outbuf,"GET / HTTP/1.0\r\n"); 
    strcat(outbuf,host_header); 
    strcat(outbuf,"Connection: close\r\n"); 
    strcat(outbuf,"\r\n"); 

    err = SSL_write(ssl, outbuf, strlen(outbuf)); 
    shutdown (sd, 1); 
    printf("(7) sent HTTP request over encrypted channel:\n\n%s\n",outbuf); 

    err = SSL_read(ssl, inbuf, sizeof(inbuf) - 1); 
    inbuf[err] = ' ';
    inbuf[err] = '\0'; 
     
    printf ("(8) got back %d bytes of HTTP response:\n\n%s\n",err,inbuf); 

    SSL_shutdown(ssl); 
    close (sd); 
    SSL_free (ssl); 
    SSL_CTX_free (ctx); 

    printf("(9) all done, cleaned up and closed connection\n\n"); 
}
 
//gcc test8.c -o test8 -lssl -lcrypto
Code:
./test8 linuxquestions.org  
                                                                                                                 
(1) SSL context initialized 

(2) 'linuxquestions.org' has IP address '35.244.195.25'

(3) TCP connection open to host 'linuxquestions.org', port 47873

(4) SSL endpoint created & handshake completed

(5) SSL connected with cipher: TLS_AES_256_GCM_SHA384

(6) server's certificate was received:

 subject: /OU=Domain Control Validated/OU=PositiveSSL/CN=www.linuxquestions.org
 issuer: /C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA

(7) sent HTTP request over encrypted channel:

GET / HTTP/1.0
Host: linuxquestions.org:443
Connection: close

(8) got back 506 bytes of HTTP response:

HTTP/1.0 301 Moved Permanently
Server: nginx
Date: Sat, 12 Nov 2022 18:43:52 GMT
Content-Type: text/html; charset=iso-8859-1
Content-Length: 238
Location: http://www.linuxquestions.org/
Via: 1.1 google
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>301 Moved Permanently</title>
</head><body>
<h1>Moved Permanently</h1>
<p>The document has moved <a href="http://www.linuxquestions.org/">here</a>.</p>
</body></html>


./test8 cloudnet.fmi.fi
(1) SSL context initialized

(2) 'cloudnet.fmi.fi' has IP address '193.166.223.85'

(3) TCP connection open to host 'cloudnet.fmi.fi', port 47873

(4) SSL endpoint created & handshake completed

(5) SSL connected with cipher: TLS_AES_256_GCM_SHA384

(6) server's certificate was received:

 subject: /C=FI/ST=Uusimaa/O=Ilmatieteen laitos/CN=*.lake.fmi.fi
 issuer: /C=NL/O=GEANT Vereniging/CN=GEANT OV RSA CA 4

(7) sent HTTP request over encrypted channel:

GET / HTTP/1.0
Host: cloudnet.fmi.fi:443
Connection: close

(8) got back 521 bytes of HTTP response:

HTTP/1.1 404 Not Found
Content-Length: 231
x-amz-request-id: tx00000000000000198b76a-00636fe9c8-38f077d-default
Accept-Ranges: bytes
Content-Type: application/xml
Date: Sat, 12 Nov 2022 18:45:28 GMT
Connection: close
X-Frame-Options: DENY
X-Content-Type-Options: nosniff
X-Xss-Protection: 1; mode=block
Content-Security-Policy:script-src: https://www.google-analytics.com https://q.quora.com
Referrer-Policy: no-referrer-when-downgrade
Strict-Transport-Security: max-age=31536000;includeSubDomains;preload


./test8 google.com
(1) SSL context initialized

(2) 'google.com' has IP address '142.250.190.78'

(3) TCP connection open to host 'google.com', port 47873

(4) SSL endpoint created & handshake completed

(5) SSL connected with cipher: TLS_AES_256_GCM_SHA384

(6) server's certificate was received:

 subject: /OU=No SNI provided; please fix your client./CN=invalid2.invalid
 issuer: /OU=No SNI provided; please fix your client./CN=invalid2.invalid

(7) sent HTTP request over encrypted channel:

GET / HTTP/1.0
Host: google.com:443
Connection: close

(8) got back 703 bytes of HTTP response:

HTTP/1.0 301 Moved Permanently
Location: https://www.google.com/
Content-Type: text/html; charset=UTF-8
Date: Sat, 12 Nov 2022 18:47:55 GMT
Expires: Mon, 12 Dec 2022 18:47:55 GMT
Cache-Control: public, max-age=2592000
Server: gws
Content-Length: 220
X-XSS-Protection: 0
X-Frame-Options: SAMEORIGIN
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"

<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
<TITLE>301 Moved</TITLE></HEAD><BODY>
<H1>301 Moved</H1>
The document has moved
<A HREF="https://www.google.com/">here</A>.
</BODY></HTML>
Maybe someone else can add more. @OP Take a close look at cloudnet.fmi.fi
 
Old 11-12-2022, 03:15 PM   #12
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
Pythons urllib will get that real fast too.

Example.py
Code:
#!/usr/bin/python 

import urllib.request

agent = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0')

user_agent = {'User-Agent': agent}

url = "https://cloudnet.fmi.fi/api/model-files?site=munich&date=2022-11-10"

file_name = "test8.html"

req = urllib.request.Request(url, data=None, headers=user_agent)

with open(file_name, 'wb') as f:
    f.write(urllib.request.urlopen(req).read())
    print(urllib.request.urlopen(req).read())
    
print('------------')
    
with urllib.request.urlopen(url) as r:
    for items in r.headers.items():
        print(items)
 
Old 11-13-2022, 09:57 AM   #13
NevemTeve
Senior Member
 
Registered: Oct 2011
Location: Budapest
Distribution: Debian/GNU/Linux, AIX
Posts: 4,387
Blog Entries: 1

Rep: Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665Reputation: 1665
I'd suggest two changes: do it without BIOs, and (more important) call SSL_set_tlsext_host_name
Attached an example.
Edit: see also this: https://en.wikipedia.org/wiki/Server_Name_Indication
Attached Files
File Type: txt testA.c.txt (4.3 KB, 4 views)

Last edited by NevemTeve; 11-14-2022 at 04:42 AM.
 
1 members found this post helpful.
Old 11-13-2022, 10:34 AM   #14
teckk
Senior Member
 
Registered: Oct 2004
Distribution: Arch
Posts: 4,448
Blog Entries: 5

Rep: Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548Reputation: 1548
Thanks for that help. I was wanting to set a user agent header for that. You also used HTTP/1.1
 
Old 11-14-2022, 04:21 AM   #15
gda
Member
 
Registered: Oct 2015
Posts: 130

Original Poster
Rep: Reputation: 26
@teckk, @NevemTeve: With your help I was able to patch my client and fix the problem. Really many thanks to both of you!!!
 
1 members found this post helpful.
  


Reply


Thread Tools Search this Thread
Search this Thread:

Advanced Search

Posting Rules
You may not post new threads
You may not post replies
You may not post attachments
You may not edit your posts

BB code is On
Smilies are On
[IMG] code is Off
HTML code is Off



Similar Threads
Thread Thread Starter Forum Replies Last Post
Yum Error: /repodata/repomd.xml: [Errno 14] HTTP Error 404: Not Found crimsonclay Linux - Server 2 04-06-2017 03:39 PM
nagios not working (http://localhost/nagios or http://ip/nagios) 404 error connect2janu Linux - Server 1 11-18-2012 03:06 AM
Getting 404 error when trying to open http://serverip/~user john_insider Linux - Server 1 12-10-2011 04:26 AM
SVN Bazaar error: Unable to handle http code 401: expected 200 or 404 for full respon vikramtheone Linux - Newbie 0 07-02-2010 09:10 AM
fc5 yum getting http 404 error mbradbones Linux - Newbie 3 02-03-2007 01:10 PM

LinuxQuestions.org > Forums > Non-*NIX Forums > Programming

All times are GMT -5. The time now is 07:09 PM.

Main Menu
Advertisement
My LQ
Write for LQ
LinuxQuestions.org is looking for people interested in writing Editorials, Articles, Reviews, and more. If you'd like to contribute content, let us know.
Main Menu
Syndicate
RSS1  Latest Threads
RSS1  LQ News
Twitter: @linuxquestions
Open Source Consulting | Domain Registration