Hi David,

I generated the certificate using WolfSSL functions such as wc_MakeSelfCert.
I posted the full snippet of certificate generation in my first post.
I closely followed your documentation to generate the certificate:
https://www.wolfssl.com/docs/wolfssl-manual/ch7/

Is some extra field required in the certificate, in order to perform mutual authentication?

Best regards

Hello David,

thank you for your reply.
Unfortunately, replacing wolfSSL_CTX_trust_peer_buffer with wolfSSL_CTX_load_verify_buffer in my client side code posted above, still produces the same -155 error.

I wasn't able to find an example in your GitHub regarding a simple client-server program with mutual authentication.
Maybe you could provide an example about this somewhere?

Also,
maybe the error could be related to my certificate?
I generated a self signed certificate using the code in my first post (following your documentation).
I use this single certificate for everything related to TLS.
It is used as both client/server certificate, and is also used as a trusted cert for mutual authentication by both sides.

Here is the content of my cert.crt file:

-----BEGIN CERTIFICATE-----
MIICnjCCAYagAwIBAgIQb2bQomyZz9M1/39tZ8vxkjANBgkqhkiG9w0BAQUFADAA
MCIYDzIwMjAxMDExMTUwOTMxWhgPMjAyMjAyMjQxNTA5MzFaMAAwggEiMA0GCSqG
SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC01kc3HnW4E+O1/MLTVpW9tMshzX+owRaf
y4CG3jNi61ApPnuKVg5iyofTPkYZUiju0ZPZtI1vLPUK94bEo+1XnKD6vLlNOtev
d2tbRjAWffVo6pD+W/SG19DBdx3alNe5dceitZ5KA1mNBz2hwjKCQbubFlmGje1O
w0HkD5X/lUuPN2bMVnvwjiq8w6NbC3d8hlzuLJCKrxBDB+3fyf671rQkd+OjXe4e
7YoIZwWncSymxmkYVh9+hrJQWdIJYQldQZwqNU1yZF43PR1k7Xd5lYL0Sh1cI5Ow
dAhYuIAs4IakdsOqrbk9vowTMhut0DeUdBR326UhdvRSWM3gQNFVAgMBAAGjEDAO
MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAEvr5cdVmCcGDpHyI6F8
FOhtDU4E9LOu7XqGPqCfvrD3iAqT/My3kxGjfuGIL1qzgb5k0bE6gSLhzsCTy/14
yEWTDjjz/crVjX1wUlOJdfm8oqswDRhVk5mjnxaNzZ1Skxdtka6m3VV7IcOMaHxB
kAwDUTFO4gwFFOLZQ6RA8O0u0hwEPkDzVbDnzqPFN/VhP8n4PZXT6ZxHD9H3IxAG
84tQs9C0rW/kwcRO/AVny/LLaf2c9xGVMi5lg3kUfrvKtiPoowV+nf11fP2x0Sns
Lt4T6aO5/bPlPh3ajd55ATs9lCdElP7mMczOdIYydOhYdt8XKKVXzZtczkJZLiS3
kNA=
-----END CERTIFICATE-----

The cert is present in the server side as a PEM file, and transferred to the client in DER format in a buffer.

Hi,

I am trying to build a simple client-server application with mutual authentication.
I want to use self-generated certificate for both client and server authentication, without having to rely on CA or third party certificates.

First, I generate my self-signed certificate.
This is the certificate to be used by both server and client, for TLS and peer verification.

To do this, I generate a RSA key, then a certificate, using wc_MakeSelfCert().

// ________________________________________________
//
//  GenerateCert
//
//  PURPOSE: 
//  Generate certificate with public & private key.
//
//  PARAMETERS:
//  None
//
//  RETURN VALUE:
//  SSL_SUCCESS If successful the call will return.
//  BAD_MUTEX_E is an error that may be returned.
//  WC_INIT_E wolfCrypt initialization error returned.
// ________________________________________________
//
Export BOOL GenerateCert()
{
    /* Generate RSA Key */

    RsaKey    genKey;
    RNG        rng;
    int     ret;
    BOOL    bRet = FALSE;
    byte    derKey[4096];
    byte    pemKey[4096];
    int        derSize;
    int        pemSize;
    HANDLE hFile;
    DWORD    dwBytesWritten;

    wc_InitRng(&rng);
    wc_InitRsaKey(&genKey, NULL);
    ret = wc_MakeRsaKey(&genKey, 2048, 65537, &rng);

    if (ret != 0) {
        /* ret contains error */;
        return FALSE;
    }

    derSize = wc_RsaKeyToDer(&genKey, derKey, sizeof(derKey));
    if (derSize < 0) {
        /* derSz contains error */;
        MessageBoxA(0, "ERROR: failed to get DER RSA Key", "", MB_ICONERROR);
        return FALSE;
    }

    pemSize = wc_DerToPem(derKey, derSize, pemKey, sizeof(pemKey), PRIVATEKEY_TYPE);
    if (pemSize < 0) {
        /* pemCertSz contains error */;
        MessageBoxA(0, "ERROR: failed to get PEM RSA Key", "", MB_ICONERROR);
        return FALSE;
    }


    /* Write Key file  */

    hFile = CreateFile(L"key.key",                // name of the write
        GENERIC_WRITE,          // open for writing
        0,                      // do not share
        NULL,                   // default security
        CREATE_ALWAYS,
        FILE_ATTRIBUTE_NORMAL,  // normal file
        NULL);

    bRet = WriteFile(
        hFile,           // open file handle
        pemKey,      // start of data to write
        pemSize,  // number of bytes to write
        &dwBytesWritten, // number of bytes that were written
        NULL);            // no overlapped structure

    CloseHandle(hFile);


    /* Generate Certificate */

    Cert myCert;
    byte derCert[4096];
    byte pemCert[4096];

    wc_InitCert(&myCert);

    derSize = wc_MakeSelfCert(&myCert, derCert, sizeof(derCert), &genKey, &rng);
    if (derSize < 0) {
        /* certSz contains the error */;
        return FALSE;
    }

    pemSize = wc_DerToPem(derCert, derSize, pemCert, sizeof(pemCert), CERT_TYPE);
    if (pemSize < 0) {
        /* pemCertSz contains error */;
        return FALSE;
    }

    /* Write Certificate file  */

    hFile = CreateFile(L"certificate.crt",                // name of the write
        GENERIC_WRITE,          // open for writing
        0,                      // do not share
        NULL,                   // default security
        CREATE_ALWAYS,
        FILE_ATTRIBUTE_NORMAL,  // normal file
        NULL);

    bRet = WriteFile(
        hFile,           // open file handle
        pemCert,      // start of data to write
        pemSize,  // number of bytes to write
        &dwBytesWritten, // number of bytes that were written
        NULL);            // no overlapped structure

    CloseHandle(hFile);

    return bRet;
}

I start my TLS server, with wolfSSL_CTX_set_verify(SSL_VERIFY_PEER) to enable peer authentication.
I also use wolfSSL_CTX_trust_peer_cert() with my self-signed certificate, so that no CA will be needed for client authentication.

// ________________________________________________
//
//    InitTLS_Server
//
//  PURPOSE: 
//  Initialize TLS Server settings:
//  + Select Cipher Suite
//  + Enable Client Authentication
//    + Load RSA Key
//
//  PARAMETERS:
//  None
//
//  RETURN VALUE:
//  Pointer to WOLFSSL_CTX on success
//  NULL on failure
// ________________________________________________
//
Export WOLFSSL_CTX* InitTLS_Server()
{
    WOLFSSL_CTX*    ctx;

    /* Create and initialize WOLFSSL_CTX */
    if ((ctx = wolfSSL_CTX_new(wolfTLSv1_3_server_method())) == NULL) {
        MessageBoxA(0, "ERROR: failed to create WOLFSSL_CTX", "", MB_ICONERROR);
        //fprintf(stderr, "ERROR: failed to create WOLFSSL_CTX\n");
        return NULL;
    }

    /* Select cipher list to use */
    if (wolfSSL_CTX_set_cipher_list(ctx, "TLS_AES_128_GCM_SHA256") != SSL_SUCCESS)
    {
        MessageBoxA(0, "ERROR: failed to set cipher list.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  /* Free the wolfSSL context object   */
        return NULL;
    }

    /* Load CA certificates into WOLFSSL_CTX */
    if (wolfSSL_CTX_load_verify_locations(ctx, CERT_FILE, NULL) != SSL_SUCCESS)
    {
        MessageBoxA(0, "ERROR: failed to load certificate.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  // Free the wolfSSL context object
        return NULL;
    }

    /* Load server certificates into WOLFSSL_CTX */
    if (wolfSSL_CTX_use_certificate_file(ctx, CERT_FILE, SSL_FILETYPE_PEM) != SSL_SUCCESS)
    {
        MessageBoxA(0, "ERROR: failed to load server certificate.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  /* Free the wolfSSL context object   */
        return NULL;
    }

    /* Load server key into WOLFSSL_CTX */
    if (wolfSSL_CTX_use_PrivateKey_file(ctx, KEY_FILE, SSL_FILETYPE_PEM)
        != SSL_SUCCESS) {
        //fprintf(stderr, "ERROR: failed to load %s, please check the file.\n", KEY_FILE);

        MessageBoxA(0, "ERROR: failed to load keyfile.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);
        return NULL;
    }

    /* Use trusted certificate to verify a peer when performing a TLS handshake (instead of CA certificate). */
    if (wolfSSL_CTX_trust_peer_cert(ctx, CERT_FILE, SSL_FILETYPE_PEM) != SSL_SUCCESS)
    {
        MessageBoxA(0, "ERROR: failed to load trusted peer certificate.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  /* Free the wolfSSL context object   */
        return NULL;
    }

    /* Enable client authentication */
    wolfSSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, 0);

    return ctx;
}

In my client side,
I load the same self-signed certificate generated earlier, using wolfSSL_CTX_use_certificate_buffer().
I use wolfSSL_CTX_trust_peer_buffer() so that no CA will be needed for server authentication.

// ________________________________________________
//
//  InitializeTLS (Client Side)
//
//  PURPOSE: 
//  One-time initialization of TLS.
//  Must be called just once at application start.
//
//  PARAMETERS:
//  Client Certificate buffer
//
//  RETURN VALUE:
//  TRUE on Success
//  FALSE on Failure
// ________________________________________________
//
bool CConnection::InitializeTLS(PBYTE pCert, int lenCert)
{
    int ret;

    /* Initialize wolfSSL */
    if ((ret = wolfSSL_Init()) != WOLFSSL_SUCCESS) {
        PrintEvent("[ERROR]", "Failed to initialize TLS");
        return false;
    }

    /* Create and initialize WOLFSSL_CTX */
    if ((ctxTLS = wolfSSL_CTX_new(wolfTLSv1_3_client_method())) == NULL) {
        PrintEvent("[ERROR]", "Failed to initialize TLS context");
        ret = -1;
        return false;
    }

    /* Load client certificates into WOLFSSL_CTX */
    if ((ret = wolfSSL_CTX_use_certificate_buffer(ctxTLS, pCert, lenCert, SSL_FILETYPE_ASN1))
        != SSL_SUCCESS) {
        PrintEvent("[ERROR]", "Failed to load TLS certificate");
        return false;
    }

    /* Select cipher list to use */
    if (wolfSSL_CTX_set_cipher_list(ctxTLS, "TLS_AES_128_GCM_SHA256") != SSL_SUCCESS)
    {
        PrintEvent("[ERROR]", "Failed to set cipher list");
        return false;
    }

    /* Use trusted certificate to verify a peer when performing a TLS handshake (instead of CA certificate). */
    if (wolfSSL_CTX_trust_peer_buffer(ctxTLS, pCert, lenCert, SSL_FILETYPE_ASN1) != SSL_SUCCESS)
    {
        PrintEvent("[ERROR]", "Failed to set peer certificate");
        return false;
    }

    bUseTLS = true;
    return true;
}

After the aforementioned initialization code (which succeeds without errors),
I connect to server using wolfSSL_connect() and I get an error on this function.
The error is: ASN no signer error to confirm failure
which seem to indicate an error regarding invalid CA certificate.
I am confused on why that happens:
both client and server are instructed with wolfSSL_CTX_trust_peer_buffer and wolfSSL_CTX_trust_peer_cert to use a custom certificate for verification.
The certificate used by the server and client is the same one, which was self-generated using the above code.

I would greatly appreciate if you can point me to the right direction.

Thank you in advance,
and Best Regards

Hello Kaleb,
thank you for your reply.

Upon further investigation,
I found out the exact lines of code from WolfSSL library which throw the error.

The code is contained in the wolfSSL_accept_TLSv13() function:
https://i.ibb.co/fSH6BSL/Screen-Shot-10-01-20-at-09-12-PM.png

It's failing because the client didn't send the "hello" message yet.
In fact, I didn't add TLS to the client yet.
When a client application does a connect() and then performs a send(), without initializing TLS first,
the send() call will trigger this error on wolfSSL_accept().

Problem solved. smile

Hello,
I am implementing TLS 1.3 to my server application (Windows).
After the winsock accept() function, I try to attach TLS to the socket using wolfSSL_accept().
I fails with "record layer length error"and I don't understand why.

I would greatly appreciate if you can point me to the right direction.

I attach here the related code with two functions:
InitTLS_Server() is the function I use to setup TLS before listen() and accept() calls. It seems to work well.
TLS_Accept() is the function which is called after accept(). It fails.


// ________________________________________________
//
//    InitTLS_Server
//
//  PURPOSE: 
//  Initialize TLS Server settings:
//  + Select Cipher Suite
//  + Enable Client Authentication
//    + Load RSA Key
//
//  PARAMETERS:
//  None
//
//  RETURN VALUE:
//  Pointer to WOLFSSL_CTX on success
//  NULL on failure
// ________________________________________________
//
Export WOLFSSL_CTX* InitTLS_Server()
{
    WOLFSSL_CTX*    ctx;

    /* Create and initialize WOLFSSL_CTX */
    if ((ctx = wolfSSL_CTX_new(wolfTLSv1_3_server_method())) == NULL) {
        fprintf(stderr, "ERROR: failed to create WOLFSSL_CTX\n");
        return NULL;
    }

    // Select cipher to use
    if (wolfSSL_CTX_set_cipher_list(ctx, "TLS_AES_128_GCM_SHA256")  != SSL_SUCCESS)
    {
        MessageBoxA(0, "ERROR: failed to set cipher list.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  /* Free the wolfSSL context object   */
        return NULL;
    }

    // Enable client authentication
    wolfSSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, 0);

    /* Load server certificates into WOLFSSL_CTX */
    if (wolfSSL_CTX_use_certificate_file(ctx, CERT_FILE, SSL_FILETYPE_PEM) != SSL_SUCCESS) 
    {
        MessageBoxA(0, "ERROR: failed to load certificate.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx);  /* Free the wolfSSL context object   */
        return NULL;
    }

    /* Load server key into WOLFSSL_CTX */
    if (wolfSSL_CTX_use_PrivateKey_file(ctx, KEY_FILE, SSL_FILETYPE_PEM)
        != SSL_SUCCESS) {
        MessageBoxA(0, "ERROR: failed to load keyfile.", "", MB_ICONERROR);
        wolfSSL_CTX_free(ctx); 
        return NULL;
    }


    return ctx;
}


// ________________________________________________
//
//    TLS_Accept
//
//  PURPOSE: 
//  Attach TLS to socket.
//
//  PARAMETERS:
//  - socket
//  - TLS Context, which contains TLS settings such as encryption type
//
//  RETURN VALUE:
//  On success: pointer to TLS Socket (WOLFSSL*)
//  On failure: NULL
// ________________________________________________
//
Export WOLFSSL* TLS_Accept(SOCKET sckt, WOLFSSL_CTX* ctx)
{
    /* declare wolfSSL objects */
    WOLFSSL*     ssl;

    int ret;

    /* Create a WOLFSSL object */
    if ((ssl = wolfSSL_new(ctx)) == NULL) {
        fprintf(stderr, "ERROR: failed to create WOLFSSL object\n");
        return NULL;
    }

    /* Attach wolfSSL to the socket */
    if (wolfSSL_set_fd(ssl, sckt) != SSL_SUCCESS) {
        MessageBoxA(0, "ERROR: wolfSSL_set_fd error", "", MB_ICONERROR);
        return NULL;
    }

    /* Establish TLS connection */
    ret = wolfSSL_accept(ssl);
    if (ret != SSL_SUCCESS) 
    {
                // ERROR IS HERE! 
                // wolfSSL_accept fails with "record layer length error"
 
        int err = wolfSSL_get_error(ssl, ret);
        char szErr[100];
        wolfSSL_ERR_error_string(err, szErr);

        MessageBoxA(0, "ERROR: wolfSSL_accept error", "", MB_ICONERROR);
        MessageBoxA(0, szErr, "", MB_ICONERROR);
        wolfSSL_free(ssl);      /* Free the wolfSSL object */
        return NULL;
    }

    printf("Client connected successfully\n");
    
    return ssl;
}