/*
 * testcode/doqclient.c - debug program. Perform multiple DNS queries using DoQ.
 *
 * Copyright (c) 2022, NLnet Labs. All rights reserved.
 *
 * This software is open source.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 
 * Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 * 
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * 
 * Neither the name of the NLNET LABS nor the names of its contributors may
 * be used to endorse or promote products derived from this software without
 * specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/** 
 * \file
 *
 * Simple DNS-over-QUIC client. For testing and debugging purposes.
 * No authentication of TLS cert.
 */

#include "config.h"
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif

#ifdef HAVE_NGTCP2
#include <ngtcp2/ngtcp2.h>
#include <ngtcp2/ngtcp2_crypto.h>
#ifdef HAVE_NGTCP2_NGTCP2_CRYPTO_OSSL_H
#include <ngtcp2/ngtcp2_crypto_ossl.h>
#elif defined(HAVE_NGTCP2_NGTCP2_CRYPTO_QUICTLS_H)
#include <ngtcp2/ngtcp2_crypto_quictls.h>
#elif defined(HAVE_NGTCP2_NGTCP2_CRYPTO_OPENSSL_H)
#include <ngtcp2/ngtcp2_crypto_openssl.h>
#define MAKE_QUIC_METHOD 1
#endif
#include <openssl/ssl.h>
#include <openssl/rand.h>
#ifdef HAVE_TIME_H
#include <time.h>
#endif
#include <sys/time.h>
#include "util/locks.h"
#include "util/net_help.h"
#include "sldns/sbuffer.h"
#include "sldns/str2wire.h"
#include "sldns/wire2str.h"
#include "util/data/msgreply.h"
#include "util/data/msgencode.h"
#include "util/data/msgparse.h"
#include "util/data/dname.h"
#include "util/random.h"
#include "util/ub_event.h"
struct doq_client_stream_list;
struct doq_client_stream;

/** the local client data for the DoQ connection */
struct doq_client_data {
        /** file descriptor */
        int fd;
        /** the event base for the events */
        struct ub_event_base* base;
        /** the ub event */
        struct ub_event* ev;
        /** the expiry timer */
        struct ub_event* expire_timer;
        /** is the expire_timer added */
        int expire_timer_added;
        /** the ngtcp2 connection information */
        struct ngtcp2_conn* conn;
        /** random state */
        struct ub_randstate* rnd;
        /** server connected to as a string */
        const char* svr;
        /** the static secret */
        uint8_t* static_secret_data;
        /** the static secret size */
        size_t static_secret_size;
        /** destination address sockaddr */
        struct sockaddr_storage dest_addr;
        /** length of dest addr */
        socklen_t dest_addr_len;
        /** local address sockaddr */
        struct sockaddr_storage local_addr;
        /** length of local addr */
        socklen_t local_addr_len;
        /** SSL context */
        SSL_CTX* ctx;
        /** SSL object */
        SSL* ssl;
#if defined(USE_NGTCP2_CRYPTO_OSSL) || defined(HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT)
        /** the connection reference for ngtcp2_conn and userdata in ssl */
        struct ngtcp2_crypto_conn_ref conn_ref;
#endif
#ifdef USE_NGTCP2_CRYPTO_OSSL
        /** the per-connection state for ngtcp2_crypto_ossl */
        struct ngtcp2_crypto_ossl_ctx* ossl_ctx;
#endif
        /** the quic version to use */
        uint32_t quic_version;
        /** the last error */
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
        struct ngtcp2_ccerr ccerr;
#else
        struct ngtcp2_connection_close_error last_error;
#endif
        /** the recent tls alert error code */
        uint8_t tls_alert;
        /** the buffer for packet operations */
        struct sldns_buffer* pkt_buf;
        /** The list of queries to start. They have no stream associated.
         * Once they do, they move to the send list. */
        struct doq_client_stream_list* query_list_start;
        /** The list of queries to send. They have a stream, and they are
         * sending data. Data could also be received, like errors. */
        struct doq_client_stream_list* query_list_send;
        /** The list of queries to receive. They have a stream, and the
         * send is done, it is possible to read data. */
        struct doq_client_stream_list* query_list_receive;
        /** The list of queries that are stopped. They have no stream
         * active any more. Write and read are done. The query is done,
         * and it may be in error and then have no answer or partial answer. */
        struct doq_client_stream_list* query_list_stop;
        /** is there a blocked packet in the blocked_pkt buffer */
        int have_blocked_pkt;
        /** store blocked packet, a packet that could not be sent on the
         * nonblocking socket. */
        struct sldns_buffer* blocked_pkt;
        /** ecn info for the blocked packet */
        struct ngtcp2_pkt_info blocked_pkt_pi;
        /** the congestion control algorithm */
        ngtcp2_cc_algo cc_algo;
        /** the transport parameters file, for early data transmission */
        const char* transport_file;
        /** the tls session file, for session resumption */
        const char* session_file;
        /** if early data is enabled for the connection */
        int early_data_enabled;
        /** how quiet is the output */
        int quiet;
        /** the configured port for the destination */
        int port;
};

/** the local client stream list, for appending streams to */
struct doq_client_stream_list {
        /** first and last members of the list */
        struct doq_client_stream* first, *last;
};

/** the local client data for a DoQ stream */
struct doq_client_stream {
        /** next stream in list, and prev in list */
        struct doq_client_stream* next, *prev;
        /** the data buffer */
        uint8_t* data;
        /** length of the data buffer */
        size_t data_len;
        /** if the client query has a stream, that is active, associated with
         * it. The stream_id is in stream_id. */
        int has_stream;
        /** the stream id */
        int64_t stream_id;
        /** data written position */
        size_t nwrite;
        /** the data length for write, in network format */
        uint16_t data_tcplen;
        /** if the write of the query data is done. That means the
         * write channel has FIN, is closed for writing. */
        int write_is_done;
        /** data read position */
        size_t nread;
        /** the answer length, in network byte order */
        uint16_t answer_len;
        /** the answer buffer */
        struct sldns_buffer* answer;
        /** the answer is complete */
        int answer_is_complete;
        /** the query has an error, it has no answer, or no complete answer */
        int query_has_error;
        /** if the query is done */
        int query_is_done;
};

#ifdef MAKE_QUIC_METHOD
/** the quic method struct, must remain valid during the QUIC connection. */
static SSL_QUIC_METHOD quic_method;
#endif

#if defined(USE_NGTCP2_CRYPTO_OSSL) || defined(HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT)
/** Get the connection ngtcp2_conn from the ssl app data
 * ngtcp2_crypto_conn_ref */
static ngtcp2_conn* conn_ref_get_conn(ngtcp2_crypto_conn_ref* conn_ref)
{
        struct doq_client_data* data = (struct doq_client_data*)
                conn_ref->user_data;
        return data->conn;
}
#endif

static void
set_app_data(SSL* ssl, struct doq_client_data* data)
{
#if defined(USE_NGTCP2_CRYPTO_OSSL) || defined(HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT)
        data->conn_ref.get_conn = &conn_ref_get_conn;
        data->conn_ref.user_data = data;
        SSL_set_app_data(ssl, &data->conn_ref);
#else
        SSL_set_app_data(ssl, data);
#endif
}

static struct doq_client_data*
get_app_data(SSL* ssl)
{
        struct doq_client_data* data;
#if defined(USE_NGTCP2_CRYPTO_OSSL) || defined(HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT)
        data = (struct doq_client_data*)((struct ngtcp2_crypto_conn_ref*)
                SSL_get_app_data(ssl))->user_data;
#else
        data = (struct doq_client_data*) SSL_get_app_data(ssl);
#endif
        return data;
}



/** write handle routine */
static void on_write(struct doq_client_data* data);
/** update the timer */
static void update_timer(struct doq_client_data* data);
/** disconnect we are done */
static void disconnect(struct doq_client_data* data);
/** fetch and write the transport file */
static void early_data_write_transport(struct doq_client_data* data);

/** usage of doqclient */
static void usage(char* argv[])
{
        printf("usage: %s [options] name type class ...\n", argv[0]);
        printf("        sends the name-type-class queries over "
                        "DNS-over-QUIC.\n");
        printf("-s server       IP address to send the queries to, "
                        "default: 127.0.0.1\n");
        printf("-p              Port to connect to, default: %d\n",
                UNBOUND_DNS_OVER_QUIC_PORT);
        printf("-v              verbose output\n");
        printf("-q              quiet, short output of answer\n");
        printf("-x file         transport file, for read/write of transport parameters.\n\t\tIf it exists, it is used to send early data. It is then\n\t\twritten to contain the last used transport parameters.\n\t\tAlso -y must be enabled for early data to succeed.\n");
        printf("-y file         session file, for read/write of TLS session. If it exists,\n\t\tit is used for TLS session resumption. It is then written\n\t\tto contain the last session used.\n\t\tOn its own, without also -x, resumes TLS session.\n");
        printf("-h              This help text\n");
        exit(1);
}

/** get the dest address */
static void
get_dest_addr(struct doq_client_data* data, const char* svr, int port)
{
        if(!ipstrtoaddr(svr, port, &data->dest_addr, &data->dest_addr_len)) {
                printf("fatal: bad server specs '%s'\n", svr);
                exit(1);
        }
}

/** open UDP socket to svr */
static int
open_svr_udp(struct doq_client_data* data)
{
        int fd = -1;
        int r;
        fd = socket(addr_is_ip6(&data->dest_addr, data->dest_addr_len)?
                PF_INET6:PF_INET, SOCK_DGRAM, 0);
        if(fd == -1) {
                perror("socket() error");
                exit(1);
        }
        r = connect(fd, (struct sockaddr*)&data->dest_addr,
                data->dest_addr_len);
        if(r < 0 && r != EINPROGRESS) {
                perror("connect() error");
                exit(1);
        }
        fd_set_nonblock(fd);
        return fd;
}

/** get the local address of the connection */
static void
get_local_addr(struct doq_client_data* data)
{
        memset(&data->local_addr, 0, sizeof(data->local_addr));
        data->local_addr_len = (socklen_t)sizeof(data->local_addr);
        if(getsockname(data->fd, (struct sockaddr*)&data->local_addr,
                &data->local_addr_len) == -1) {
                perror("getsockname() error");
                exit(1);
        }
        log_addr(1, "local_addr", &data->local_addr, data->local_addr_len);
        log_addr(1, "dest_addr", &data->dest_addr, data->dest_addr_len);
}

static sldns_buffer*
make_query(char* qname, char* qtype, char* qclass)
{
        struct query_info qinfo;
        struct edns_data edns;
        sldns_buffer* buf = sldns_buffer_new(65553);
        if(!buf) fatal_exit("out of memory");
        qinfo.qname = sldns_str2wire_dname(qname, &qinfo.qname_len);
        if(!qinfo.qname) {
                printf("cannot parse query name: '%s'\n", qname);
                exit(1);
        }

        qinfo.qtype = sldns_get_rr_type_by_name(qtype);
        qinfo.qclass = sldns_get_rr_class_by_name(qclass);
        qinfo.local_alias = NULL;

        qinfo_query_encode(buf, &qinfo); /* flips buffer */
        free(qinfo.qname);
        sldns_buffer_write_u16_at(buf, 0, 0x0000);
        sldns_buffer_write_u16_at(buf, 2, BIT_RD);
        memset(&edns, 0, sizeof(edns));
        edns.edns_present = 1;
        edns.bits = EDNS_DO;
        edns.udp_size = 4096;
        if(sldns_buffer_capacity(buf) >=
                sldns_buffer_limit(buf)+calc_edns_field_size(&edns))
                attach_edns_record(buf, &edns);
        return buf;
}

/** create client stream structure */
static struct doq_client_stream*
client_stream_create(struct sldns_buffer* query_data)
{
        struct doq_client_stream* str = calloc(1, sizeof(*str));
        if(!str)
                fatal_exit("calloc failed: out of memory");
        str->data = memdup(sldns_buffer_begin(query_data),
                sldns_buffer_limit(query_data));
        if(!str->data)
                fatal_exit("alloc data failed: out of memory");
        str->data_len = sldns_buffer_limit(query_data);
        str->stream_id = -1;
        return str;
}

/** free client stream structure */
static void
client_stream_free(struct doq_client_stream* str)
{
        if(!str)
                return;
        free(str->data);
        sldns_buffer_free(str->answer);
        free(str);
}

/** setup the stream to start the write process */
static void
client_stream_start_setup(struct doq_client_stream* str, int64_t stream_id)
{
        str->has_stream = 1;
        str->stream_id = stream_id;
        str->nwrite = 0;
        str->nread = 0;
        str->answer_len = 0;
        str->query_is_done = 0;
        str->answer_is_complete = 0;
        str->query_has_error = 0;
        if(str->answer) {
                sldns_buffer_free(str->answer);
                str->answer = NULL;
        }
}

/** Return string for log purposes with query name. */
static char*
client_stream_string(struct doq_client_stream* str)
{
        char* s;
        size_t dname_len;
        char dname[256], tpstr[32], result[256+32+16];
        uint16_t tp;
        if(str->data_len <= LDNS_HEADER_SIZE) {
                s = strdup("query_with_no_question");
                if(!s)
                        fatal_exit("strdup failed: out of memory");
                return s;
        }
        dname_len = dname_valid(str->data+LDNS_HEADER_SIZE,
                str->data_len-LDNS_HEADER_SIZE);
        if(!dname_len) {
                s = strdup("query_dname_not_valid");
                if(!s)
                        fatal_exit("strdup failed: out of memory");
                return s;
        }
        (void)sldns_wire2str_dname_buf(str->data+LDNS_HEADER_SIZE, dname_len,
                dname, sizeof(dname));
        tp = sldns_wirerr_get_type(str->data+LDNS_HEADER_SIZE,
                str->data_len-LDNS_HEADER_SIZE, dname_len);
        (void)sldns_wire2str_type_buf(tp, tpstr, sizeof(tpstr));
        snprintf(result, sizeof(result), "%s %s", dname, tpstr);
        s = strdup(result);
        if(!s)
                fatal_exit("strdup failed: out of memory");
        return s;
}

/** create query stream list */
static struct doq_client_stream_list*
stream_list_create(void)
{
        struct doq_client_stream_list* list = calloc(1, sizeof(*list));
        if(!list)
                fatal_exit("calloc failed: out of memory");
        return list;
}

/** free the query stream list */
static void
stream_list_free(struct doq_client_stream_list* list)
{
        struct doq_client_stream* str;
        if(!list)
                return;
        str = list->first;
        while(str) {
                struct doq_client_stream* next = str->next;
                client_stream_free(str);
                str = next;
        }
        free(list);
}

/** append item to list */
static void
stream_list_append(struct doq_client_stream_list* list,
        struct doq_client_stream* str)
{
        if(list->last) {
                str->prev = list->last;
                list->last->next = str;
        } else {
                str->prev = NULL;
                list->first = str;
        }
        str->next = NULL;
        list->last = str;
}

/** delete the item from the list */
static void
stream_list_delete(struct doq_client_stream_list* list,
        struct doq_client_stream* str)
{
        if(str->next) {
                str->next->prev = str->prev;
        } else {
                list->last = str->prev;
        }
        if(str->prev) {
                str->prev->next = str->next;
        } else {
                list->first = str->next;
        }
        str->prev = NULL;
        str->next = NULL;
}

/** move the item from list1 to list2 */
static void
stream_list_move(struct doq_client_stream* str,
        struct doq_client_stream_list* list1,
        struct doq_client_stream_list* list2)
{
        stream_list_delete(list1, str);
        stream_list_append(list2, str);
}

/** allocate stream data buffer, then answer length is complete */
static void
client_stream_datalen_complete(struct doq_client_stream* str)
{
        verbose(1, "answer length %d", (int)ntohs(str->answer_len));
        str->answer = sldns_buffer_new(ntohs(str->answer_len));
        if(!str->answer)
                fatal_exit("sldns_buffer_new failed: out of memory");
        sldns_buffer_set_limit(str->answer, ntohs(str->answer_len));
}

/** print the answer rrs */
static void
print_answer_rrs(uint8_t* pkt, size_t pktlen)
{
        char buf[65535];
        char* str;
        size_t str_len;
        int i, qdcount, ancount;
        uint8_t* data = pkt;
        size_t data_len = pktlen;
        int comprloop = 0;
        if(data_len < LDNS_HEADER_SIZE)
                return;
        qdcount = LDNS_QDCOUNT(data);
        ancount = LDNS_ANCOUNT(data);
        data += LDNS_HEADER_SIZE;
        data_len -= LDNS_HEADER_SIZE;

        for(i=0; i<qdcount; i++) {
                str = buf;
                str_len = sizeof(buf);
                (void)sldns_wire2str_rrquestion_scan(&data, &data_len,
                        &str, &str_len, pkt, pktlen, &comprloop);
        }
        for(i=0; i<ancount; i++) {
                str = buf;
                str_len = sizeof(buf);
                (void)sldns_wire2str_rr_scan(&data, &data_len, &str, &str_len,
                        pkt, pktlen, &comprloop);
                /* terminate string */
                if(str_len == 0)
                        buf[sizeof(buf)-1] = 0;
                else    *str = 0;
                printf("%s", buf);
        }
}

/** short output of answer, short error or rcode or answer section RRs. */
static void
client_stream_print_short(struct doq_client_stream* str)
{
        int rcode, ancount;
        if(str->query_has_error) {
                char* logs = client_stream_string(str);
                printf("%s has error, there is no answer\n", logs);
                free(logs);
                return;
        }
        if(sldns_buffer_limit(str->answer) < LDNS_HEADER_SIZE) {
                char* logs = client_stream_string(str);
                printf("%s received short packet, smaller than header\n",
                        logs);
                free(logs);
                return;
        }
        rcode = LDNS_RCODE_WIRE(sldns_buffer_begin(str->answer));
        if(rcode != 0) {
                char* logs = client_stream_string(str);
                char rc[16];
                (void)sldns_wire2str_rcode_buf(rcode, rc, sizeof(rc));
                printf("%s rcode %s\n", logs, rc);
                free(logs);
                return;
        }
        ancount = LDNS_ANCOUNT(sldns_buffer_begin(str->answer));
        if(ancount == 0) {
                char* logs = client_stream_string(str);
                printf("%s nodata answer\n", logs);
                free(logs);
                return;
        }
        print_answer_rrs(sldns_buffer_begin(str->answer),
                sldns_buffer_limit(str->answer));
}

/** print the stream output answer */
static void
client_stream_print_long(struct doq_client_data* data,
        struct doq_client_stream* str)
{
        char* s;
        if(str->query_has_error) {
                char* logs = client_stream_string(str);
                printf("%s has error, there is no answer\n", logs);
                free(logs);
                return;
        }
        s = sldns_wire2str_pkt(sldns_buffer_begin(str->answer),
                sldns_buffer_limit(str->answer));
        printf("%s", (s?s:";sldns_wire2str_pkt failed\n"));
        printf(";; SERVER: %s %d\n", data->svr, data->port);
        free(s);
}

/** the stream has completed the data */
static void
client_stream_data_complete(struct doq_client_stream* str)
{
        verbose(1, "received all answer content");
        if(verbosity > 0) {
                char* logs = client_stream_string(str);
                char* s;
                log_buf(1, "received answer", str->answer);
                s = sldns_wire2str_pkt(sldns_buffer_begin(str->answer),
                        sldns_buffer_limit(str->answer));
                if(!s) verbose(1, "could not sldns_wire2str_pkt");
                else verbose(1, "query %s received:\n%s", logs, s);
                free(s);
                free(logs);
        }
        str->answer_is_complete = 1;
}

/** the stream has completed but with an error */
static void
client_stream_answer_error(struct doq_client_stream* str)
{
        if(verbosity > 0) {
                char* logs = client_stream_string(str);
                if(str->answer)
                        verbose(1, "query %s has an error. received %d/%d bytes.",
                                logs, (int)sldns_buffer_position(str->answer),
                                (int)sldns_buffer_limit(str->answer));
                else
                        verbose(1, "query %s has an error. received no data.",
                                logs);
                free(logs);
        }
        str->query_has_error = 1;
}

/** receive data for a stream */
static void
client_stream_recv_data(struct doq_client_stream* str, const uint8_t* data,
        size_t datalen)
{
        int got_data = 0;
        /* read the tcplength uint16_t at the start of the DNS message */
        if(str->nread < 2) {
                size_t to_move = datalen;
                if(datalen > 2-str->nread)
                        to_move = 2-str->nread;
                memmove(((uint8_t*)&str->answer_len)+str->nread, data,
                        to_move);
                str->nread += to_move;
                data += to_move;
                datalen -= to_move;
                if(str->nread == 2) {
                        /* we can allocate the data buffer */
                        client_stream_datalen_complete(str);
                }
        }
        /* if we have data bytes */
        if(datalen > 0) {
                size_t to_write = datalen;
                if(datalen > sldns_buffer_remaining(str->answer))
                        to_write = sldns_buffer_remaining(str->answer);
                if(to_write > 0) {
                        sldns_buffer_write(str->answer, data, to_write);
                        str->nread += to_write;
                        data += to_write;
                        datalen -= to_write;
                        got_data = 1;
                }
        }
        /* extra received bytes after end? */
        if(datalen > 0) {
                verbose(1, "extra bytes after end of DNS length");
                if(verbosity > 0)
                        log_hex("extradata", (void*)data, datalen);
        }
        /* are we done with it? */
        if(got_data && str->nread >= (size_t)(ntohs(str->answer_len))+2) {
                client_stream_data_complete(str);
        }
}

/** receive FIN from remote end on client stream, no more data to be
 * received on the stream. */
static void
client_stream_recv_fin(struct doq_client_data* data,
        struct doq_client_stream* str, int is_fin)
{
        if(verbosity > 0) {
                char* logs = client_stream_string(str);
                if(is_fin)
                        verbose(1, "query %s: received FIN from remote", logs);
                else
                        verbose(1, "query %s: stream reset from remote", logs);
                free(logs);
        }
        if(str->write_is_done)
                stream_list_move(str, data->query_list_receive,
                        data->query_list_stop);
        else
                stream_list_move(str, data->query_list_send,
                        data->query_list_stop);
        if(!str->answer_is_complete) {
                client_stream_answer_error(str);
        }
        str->query_is_done = 1;
        if(data->quiet)
                client_stream_print_short(str);
        else client_stream_print_long(data, str);
        if(data->query_list_send->first==NULL &&
                data->query_list_receive->first==NULL)
                disconnect(data);
}

/** fill a buffer with random data */
static void fill_rand(struct ub_randstate* rnd, uint8_t* buf, size_t len)
{
        if(RAND_bytes(buf, len) != 1) {
                size_t i;
                for(i=0; i<len; i++)
                        buf[i] = ub_random(rnd)&0xff;
        }
}

/** create the static secret */
static void generate_static_secret(struct doq_client_data* data, size_t len)
{
        data->static_secret_data = malloc(len);
        if(!data->static_secret_data)
                fatal_exit("malloc failed: out of memory");
        data->static_secret_size = len;
        fill_rand(data->rnd, data->static_secret_data, len);
}

/** fill cid structure with random data */
static void cid_randfill(struct ngtcp2_cid* cid, size_t datalen,
        struct ub_randstate* rnd)
{
        uint8_t buf[32];
        if(datalen > sizeof(buf))
                datalen = sizeof(buf);
        fill_rand(rnd, buf, datalen);
        ngtcp2_cid_init(cid, buf, datalen);
}

/** send buf on the client stream */
static int
client_bidi_stream(struct doq_client_data* data, int64_t* ret_stream_id,
        void* stream_user_data)
{
        int64_t stream_id;
        int rv;

        /* open new bidirectional stream */
        rv = ngtcp2_conn_open_bidi_stream(data->conn, &stream_id,
                stream_user_data);
        if(rv != 0) {
                if(rv == NGTCP2_ERR_STREAM_ID_BLOCKED) {
                        /* no bidi stream count for this new stream */
                        return 0;
                }
                fatal_exit("could not ngtcp2_conn_open_bidi_stream: %s",
                        ngtcp2_strerror(rv));
        }
        *ret_stream_id = stream_id;
        return 1;
}

/** See if we can start query streams, by creating bidirectional streams
 * on the QUIC transport for them. */
static void
query_streams_start(struct doq_client_data* data)
{
        while(data->query_list_start->first) {
                struct doq_client_stream* str = data->query_list_start->first;
                int64_t stream_id = 0;
                if(!client_bidi_stream(data, &stream_id, str)) {
                        /* no more bidi streams allowed */
                        break;
                }
                if(verbosity > 0) {
                        char* logs = client_stream_string(str);
                        verbose(1, "query %s start on bidi stream id %lld",
                                logs, (long long int)stream_id);
                        free(logs);
                }
                /* setup the stream to start */
                client_stream_start_setup(str, stream_id);
                /* move the query entry to the send list to write it */
                stream_list_move(str, data->query_list_start,
                        data->query_list_send);
        }
}

/** the rand callback routine from ngtcp2 */
static void rand_cb(uint8_t* dest, size_t destlen,
        const ngtcp2_rand_ctx* rand_ctx)
{
        struct ub_randstate* rnd = (struct ub_randstate*)
                rand_ctx->native_handle;
        fill_rand(rnd, dest, destlen);
}

/** the get_new_connection_id callback routine from ngtcp2 */
static int get_new_connection_id_cb(struct ngtcp2_conn* ATTR_UNUSED(conn),
        struct ngtcp2_cid* cid, uint8_t* token, size_t cidlen, void* user_data)
{
        struct doq_client_data* data = (struct doq_client_data*)user_data;
        cid_randfill(cid, cidlen, data->rnd);
        if(ngtcp2_crypto_generate_stateless_reset_token(token,
                data->static_secret_data, data->static_secret_size, cid) != 0)
                return NGTCP2_ERR_CALLBACK_FAILURE;
        return 0;
}

/** handle that early data is rejected */
static void
early_data_is_rejected(struct doq_client_data* data)
{
        int rv;
        verbose(1, "early data was rejected by the server");
#ifdef HAVE_NGTCP2_CONN_TLS_EARLY_DATA_REJECTED
        rv = ngtcp2_conn_tls_early_data_rejected(data->conn);
#else
        rv = ngtcp2_conn_early_data_rejected(data->conn);
#endif
        if(rv != 0) {
                log_err("ngtcp2_conn_early_data_rejected failed: %s",
                        ngtcp2_strerror(rv));
                return;
        }
        /* move the streams back to the start state */
        while(data->query_list_send->first) {
                struct doq_client_stream* str = data->query_list_send->first;
                /* move it back to the start list */
                stream_list_move(str, data->query_list_send,
                        data->query_list_start);
                str->has_stream = 0;
                /* remove stream id */
                str->stream_id = 0;
                /* initialise other members, in case they are altered,
                 * but unlikely, because early streams are rejected. */
                str->nwrite = 0;
                str->nread = 0;
                str->answer_len = 0;
                str->query_is_done = 0;
                str->answer_is_complete = 0;
                str->query_has_error = 0;
                if(str->answer) {
                        sldns_buffer_free(str->answer);
                        str->answer = NULL;
                }
        }
}

/** the handshake completed callback from ngtcp2 */
static int
handshake_completed(ngtcp2_conn* ATTR_UNUSED(conn), void* user_data)
{
        struct doq_client_data* data = (struct doq_client_data*)user_data;
        verbose(1, "handshake_completed callback");
        verbose(1, "ngtcp2_conn_get_max_data_left is %d",
                (int)ngtcp2_conn_get_max_data_left(data->conn));
#ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
        verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
                (int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
#endif
        verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
                (int)ngtcp2_conn_get_streams_uni_left(data->conn));
        verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
                (int)ngtcp2_conn_get_streams_bidi_left(data->conn));
        verbose(1, "negotiated cipher name is %s",
                SSL_get_cipher_name(data->ssl));
        if(verbosity > 0) {
                const unsigned char* alpn = NULL;
                unsigned int alpnlen = 0;
                char alpnstr[128];
                SSL_get0_alpn_selected(data->ssl, &alpn, &alpnlen);
                if(alpnlen > sizeof(alpnstr)-1)
                        alpnlen = sizeof(alpnstr)-1;
                memmove(alpnstr, alpn, alpnlen);
                alpnstr[alpnlen]=0;
                verbose(1, "negotiated ALPN is '%s'", alpnstr);
        }
        /* The SSL_get_early_data_status call works after the handshake
         * completes. */
        if(data->early_data_enabled) {
                if(SSL_get_early_data_status(data->ssl) !=
                        SSL_EARLY_DATA_ACCEPTED) {
                        early_data_is_rejected(data);
                } else {
                        verbose(1, "early data was accepted by the server");
                }
        }
#if defined(USE_NGTCP2_CRYPTO_OSSL) || defined(HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT)
        if(data->transport_file) {
                early_data_write_transport(data);
        }
#endif
        return 0;
}

/** the extend_max_local_streams_bidi callback from ngtcp2 */
static int
extend_max_local_streams_bidi(ngtcp2_conn* ATTR_UNUSED(conn),
        uint64_t max_streams, void* user_data)
{
        struct doq_client_data* data = (struct doq_client_data*)user_data;
        verbose(1, "extend_max_local_streams_bidi callback, %d max_streams",
                (int)max_streams);
        verbose(1, "ngtcp2_conn_get_max_data_left is %d",
                (int)ngtcp2_conn_get_max_data_left(data->conn));
#ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
        verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
                (int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
#endif
        verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
                (int)ngtcp2_conn_get_streams_uni_left(data->conn));
        verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
                (int)ngtcp2_conn_get_streams_bidi_left(data->conn));
        query_streams_start(data);
        return 0;
}

/** the recv_stream_data callback from ngtcp2 */
static int
recv_stream_data(ngtcp2_conn* ATTR_UNUSED(conn), uint32_t flags,
        int64_t stream_id, uint64_t offset, const uint8_t* data,
        size_t datalen, void* user_data, void* stream_user_data)
{
        struct doq_client_data* doqdata = (struct doq_client_data*)user_data;
        struct doq_client_stream* str = (struct doq_client_stream*)
                stream_user_data;
        verbose(1, "recv_stream_data stream %d offset %d datalen %d%s%s",
                (int)stream_id, (int)offset, (int)datalen,
                ((flags&NGTCP2_STREAM_DATA_FLAG_FIN)!=0?" FIN":""),
#ifdef NGTCP2_STREAM_DATA_FLAG_0RTT
                ((flags&NGTCP2_STREAM_DATA_FLAG_0RTT)!=0?" 0RTT":"")
#else
                ((flags&NGTCP2_STREAM_DATA_FLAG_EARLY)!=0?" EARLY":"")
#endif
                );
        if(verbosity > 0)
                log_hex("data", (void*)data, datalen);
        if(verbosity > 0) {
                char* logs = client_stream_string(str);
                verbose(1, "the stream_user_data is %s stream id %d, nread %d",
                        logs, (int)str->stream_id, (int)str->nread);
                free(logs);
        }

        /* append the data, if there is data */
        if(datalen > 0) {
                client_stream_recv_data(str, data, datalen);
        }
        if((flags&NGTCP2_STREAM_DATA_FLAG_FIN)!=0) {
                client_stream_recv_fin(doqdata, str, 1);
        }
        ngtcp2_conn_extend_max_stream_offset(doqdata->conn, stream_id, datalen);
        ngtcp2_conn_extend_max_offset(doqdata->conn, datalen);
        return 0;
}

/** the stream reset callback from ngtcp2 */
static int
stream_reset(ngtcp2_conn* ATTR_UNUSED(conn), int64_t stream_id,
        uint64_t final_size, uint64_t app_error_code, void* user_data,
        void* stream_user_data)
{
        struct doq_client_data* doqdata = (struct doq_client_data*)user_data;
        struct doq_client_stream* str = (struct doq_client_stream*)
                stream_user_data;
        verbose(1, "stream reset for stream %d final size %d app error code %d",
                (int)stream_id, (int)final_size, (int)app_error_code);
        client_stream_recv_fin(doqdata, str, 0);
        return 0;
}

/** copy sockaddr into ngtcp2 addr */
static void
copy_ngaddr(struct ngtcp2_addr* ngaddr, struct sockaddr_storage* addr,
        socklen_t addrlen)
{
        if(addr_is_ip6(addr, addrlen)) {
#if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
                struct sockaddr_in* i6 = (struct sockaddr_in6*)addr;
                struct ngtcp2_sockaddr_in6 a6;
                ngaddr->addr = calloc(1, sizeof(a6));
                if(!ngaddr->addr) fatal_exit("calloc failed: out of memory");
                ngaddr->addrlen = sizeof(a6);
                memset(&a6, 0, sizeof(a6));
                a6.sin6_family = i6->sin6_family;
                a6.sin6_port = i6->sin6_port;
                a6.sin6_flowinfo = i6->sin6_flowinfo;
                memmove(&a6.sin6_addr, i6->sin6_addr, sizeof(a6.sin6_addr);
                a6.sin6_scope_id = i6->sin6_scope_id;
                memmove(ngaddr->addr, &a6, sizeof(a6));
#else
                ngaddr->addr = (ngtcp2_sockaddr*)addr;
                ngaddr->addrlen = addrlen;
#endif
        } else {
#ifdef NGTCP2_USE_GENERIC_SOCKADDR
                struct sockaddr_in* i4 = (struct sockaddr_in*)addr;
                struct ngtcp2_sockaddr_in a4;
                ngaddr->addr = calloc(1, sizeof(a4));
                if(!ngaddr->addr) fatal_exit("calloc failed: out of memory");
                ngaddr->addrlen = sizeof(a4);
                memset(&a4, 0, sizeof(a4));
                a4.sin_family = i4->sin_family;
                a4.sin_port = i4->sin_port;
                memmove(&a4.sin_addr, i4->sin_addr, sizeof(a4.sin_addr);
                memmove(ngaddr->addr, &a4, sizeof(a4));
#else
                ngaddr->addr = (ngtcp2_sockaddr*)addr;
                ngaddr->addrlen = addrlen;
#endif
        }
}

/** debug log printf for ngtcp2 connections */
static void log_printf_for_doq(void* ATTR_UNUSED(user_data),
        const char* fmt, ...)
{
        va_list ap;
        va_start(ap, fmt);
        fprintf(stderr, "libngtcp2: ");
        vfprintf(stderr, fmt, ap);
        va_end(ap);
        fprintf(stderr, "\n");
}

/** get a timestamp in nanoseconds */
static ngtcp2_tstamp get_timestamp_nanosec(void)
{
#ifdef CLOCK_REALTIME
        struct timespec tp;
        memset(&tp, 0, sizeof(tp));
#ifdef CLOCK_MONOTONIC
        if(clock_gettime(CLOCK_MONOTONIC, &tp) == -1) {
#endif
                if(clock_gettime(CLOCK_REALTIME, &tp) == -1) {
                        log_err("clock_gettime failed: %s", strerror(errno));
                }
#ifdef CLOCK_MONOTONIC
        }
#endif
        return ((uint64_t)tp.tv_sec)*((uint64_t)1000000000) +
                ((uint64_t)tp.tv_nsec);
#else
        struct timeval tv;
        if(gettimeofday(&tv, NULL) < 0) {
                log_err("gettimeofday failed: %s", strerror(errno));
        }
        return ((uint64_t)tv.tv_sec)*((uint64_t)1000000000) +
                ((uint64_t)tv.tv_usec)*((uint64_t)1000);
#endif /* CLOCK_REALTIME */
}

/** create ngtcp2 client connection and set up. */
static struct ngtcp2_conn* conn_client_setup(struct doq_client_data* data)
{
        struct ngtcp2_conn* conn = NULL;
        int rv;
        struct ngtcp2_cid dcid, scid;
        struct ngtcp2_path path;
        uint32_t client_chosen_version = NGTCP2_PROTO_VER_V1;
        struct ngtcp2_callbacks cbs;
        struct ngtcp2_settings settings;
        struct ngtcp2_transport_params params;

        memset(&cbs, 0, sizeof(cbs));
        memset(&settings, 0, sizeof(settings));
        memset(&params, 0, sizeof(params));
        memset(&dcid, 0, sizeof(dcid));
        memset(&scid, 0, sizeof(scid));
        memset(&path, 0, sizeof(path));

        data->quic_version = client_chosen_version;
        ngtcp2_settings_default(&settings);
        if(str_is_ip6(data->svr)) {
#ifdef HAVE_STRUCT_NGTCP2_SETTINGS_MAX_TX_UDP_PAYLOAD_SIZE
                settings.max_tx_udp_payload_size = 1232;
#else
                settings.max_udp_payload_size = 1232;
#endif
        }
        settings.rand_ctx.native_handle = data->rnd;
        if(verbosity > 0) {
                /* make debug logs */
                settings.log_printf = log_printf_for_doq;
        }
        settings.initial_ts = get_timestamp_nanosec();
        ngtcp2_transport_params_default(&params);
        params.initial_max_stream_data_bidi_local = 256*1024;
        params.initial_max_stream_data_bidi_remote = 256*1024;
        params.initial_max_stream_data_uni = 256*1024;
        params.initial_max_data = 1024*1024;
        params.initial_max_streams_bidi = 0;
        params.initial_max_streams_uni = 100;
        params.max_idle_timeout = 30*NGTCP2_SECONDS;
        params.active_connection_id_limit = 7;
        cid_randfill(&dcid, 16, data->rnd);
        cid_randfill(&scid, 16, data->rnd);
        cbs.client_initial = ngtcp2_crypto_client_initial_cb;
        cbs.recv_crypto_data = ngtcp2_crypto_recv_crypto_data_cb;
        cbs.encrypt = ngtcp2_crypto_encrypt_cb;
        cbs.decrypt = ngtcp2_crypto_decrypt_cb;
        cbs.hp_mask = ngtcp2_crypto_hp_mask_cb;
        cbs.recv_retry = ngtcp2_crypto_recv_retry_cb;
        cbs.update_key = ngtcp2_crypto_update_key_cb;
        cbs.delete_crypto_aead_ctx = ngtcp2_crypto_delete_crypto_aead_ctx_cb;
        cbs.delete_crypto_cipher_ctx =
                ngtcp2_crypto_delete_crypto_cipher_ctx_cb;
        cbs.get_path_challenge_data = ngtcp2_crypto_get_path_challenge_data_cb;
        cbs.version_negotiation = ngtcp2_crypto_version_negotiation_cb;
        cbs.get_new_connection_id = get_new_connection_id_cb;
        cbs.handshake_completed = handshake_completed;
        cbs.extend_max_local_streams_bidi = extend_max_local_streams_bidi;
        cbs.rand = rand_cb;
        cbs.recv_stream_data = recv_stream_data;
        cbs.stream_reset = stream_reset;
        copy_ngaddr(&path.local, &data->local_addr, data->local_addr_len);
        copy_ngaddr(&path.remote, &data->dest_addr, data->dest_addr_len);

        rv = ngtcp2_conn_client_new(&conn, &dcid, &scid, &path,
                client_chosen_version, &cbs, &settings, &params,
                NULL, /* ngtcp2_mem allocator, use default */
                data /* callback argument */);
        if(!conn) fatal_exit("could not ngtcp2_conn_client_new: %s",
                ngtcp2_strerror(rv));
        data->cc_algo = settings.cc_algo;
        return conn;
}

#ifndef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
/** write the transport file */
static void
transport_file_write(const char* file, struct ngtcp2_transport_params* params)
{
        FILE* out;
        out = fopen(file, "w");
        if(!out) {
                perror(file);
                return;
        }
        fprintf(out, "initial_max_streams_bidi=%u\n",
                (unsigned)params->initial_max_streams_bidi);
        fprintf(out, "initial_max_streams_uni=%u\n",
                (unsigned)params->initial_max_streams_uni);
        fprintf(out, "initial_max_stream_data_bidi_local=%u\n",
                (unsigned)params->initial_max_stream_data_bidi_local);
        fprintf(out, "initial_max_stream_data_bidi_remote=%u\n",
                (unsigned)params->initial_max_stream_data_bidi_remote);
        fprintf(out, "initial_max_stream_data_uni=%u\n",
                (unsigned)params->initial_max_stream_data_uni);
        fprintf(out, "initial_max_data=%u\n",
                (unsigned)params->initial_max_data);
        fprintf(out, "active_connection_id_limit=%u\n",
                (unsigned)params->active_connection_id_limit);
        fprintf(out, "max_datagram_frame_size=%u\n",
                (unsigned)params->max_datagram_frame_size);
        if(ferror(out)) {
                verbose(1, "There was an error writing %s: %s",
                        file, strerror(errno));
                fclose(out);
                return;
        }
        fclose(out);
}
#endif /* HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS */

/** fetch and write the transport file */
static void
early_data_write_transport(struct doq_client_data* data)
{
#ifdef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
        FILE* out;
        uint8_t buf[1024];
        ngtcp2_ssize len = ngtcp2_conn_encode_0rtt_transport_params(data->conn,
                buf, sizeof(buf));
        if(len < 0) {
                log_err("ngtcp2_conn_encode_0rtt_transport_params failed: %s",
                        ngtcp2_strerror(len));
                return;
        }
        out = fopen(data->transport_file, "w");
        if(!out) {
                perror(data->transport_file);
                return;
        }
        if(fwrite(buf, 1, len, out) != (size_t)len) {
                log_err("fwrite %s failed: %s", data->transport_file,
                        strerror(errno));
        }
        if(ferror(out)) {
                verbose(1, "There was an error writing %s: %s",
                        data->transport_file, strerror(errno));
        }
        fclose(out);
#else
        struct ngtcp2_transport_params params;
        memset(&params, 0, sizeof(params));
        ngtcp2_conn_get_remote_transport_params(data->conn, &params);
        transport_file_write(data->transport_file, &params);
#endif
}

#ifdef MAKE_QUIC_METHOD
/** applicatation rx key callback, this is where the rx key is set,
 * and streams can be opened, like http3 unidirectional streams, like
 * the http3 control and http3 qpack encode and decoder streams. */
static int
application_rx_key_cb(struct doq_client_data* data)
{
        verbose(1, "application_rx_key_cb callback");
        verbose(1, "ngtcp2_conn_get_max_data_left is %d",
                (int)ngtcp2_conn_get_max_data_left(data->conn));
#ifdef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
        verbose(1, "ngtcp2_conn_get_max_local_streams_uni is %d",
                (int)ngtcp2_conn_get_max_local_streams_uni(data->conn));
#endif
        verbose(1, "ngtcp2_conn_get_streams_uni_left is %d",
                (int)ngtcp2_conn_get_streams_uni_left(data->conn));
        verbose(1, "ngtcp2_conn_get_streams_bidi_left is %d",
                (int)ngtcp2_conn_get_streams_bidi_left(data->conn));
        if(data->transport_file) {
                early_data_write_transport(data);
        }
        return 1;
}

/** quic_method set_encryption_secrets function */
static int
set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
        const uint8_t *read_secret, const uint8_t *write_secret,
        size_t secret_len)
{
        struct doq_client_data* data = get_app_data(ssl);
#ifdef HAVE_NGTCP2_ENCRYPTION_LEVEL
        ngtcp2_encryption_level
#else
        ngtcp2_crypto_level
#endif
                level =
#ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_FROM_OSSL_ENCRYPTION_LEVEL
                ngtcp2_crypto_quictls_from_ossl_encryption_level(ossl_level);
#else
                ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
#endif

        if(read_secret) {
                if(ngtcp2_crypto_derive_and_install_rx_key(data->conn, NULL,
                        NULL, NULL, level, read_secret, secret_len) != 0) {
                        log_err("ngtcp2_crypto_derive_and_install_rx_key failed");
                        return 0;
                }
                if(level == NGTCP2_CRYPTO_LEVEL_APPLICATION) {
                        if(!application_rx_key_cb(data))
                                return 0;
                }
        }

        if(write_secret) {
                if(ngtcp2_crypto_derive_and_install_tx_key(data->conn, NULL,
                        NULL, NULL, level, write_secret, secret_len) != 0) {
                        log_err("ngtcp2_crypto_derive_and_install_tx_key failed");
                        return 0;
                }
        }
        return 1;
}

/** quic_method add_handshake_data function */
static int
add_handshake_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
        const uint8_t *data, size_t len)
{
        struct doq_client_data* doqdata = get_app_data(ssl);
#ifdef HAVE_NGTCP2_ENCRYPTION_LEVEL
        ngtcp2_encryption_level
#else
        ngtcp2_crypto_level
#endif
                level =
#ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_FROM_OSSL_ENCRYPTION_LEVEL
                ngtcp2_crypto_quictls_from_ossl_encryption_level(ossl_level);
#else
                ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
#endif
        int rv;

        rv = ngtcp2_conn_submit_crypto_data(doqdata->conn, level, data, len);
        if(rv != 0) {
                log_err("ngtcp2_conn_submit_crypto_data failed: %s",
                        ngtcp2_strerror(rv));
                ngtcp2_conn_set_tls_error(doqdata->conn, rv);
                return 0;
        }
        return 1;
}

/** quic_method flush_flight function */
static int
flush_flight(SSL* ATTR_UNUSED(ssl))
{
        return 1;
}

/** quic_method send_alert function */
static int
send_alert(SSL *ssl, enum ssl_encryption_level_t ATTR_UNUSED(level),
        uint8_t alert)
{
        struct doq_client_data* data = get_app_data(ssl);
        data->tls_alert = alert;
        return 1;
}
#endif /* MAKE_QUIC_METHOD */

/** new session callback. We can write it to file for resumption later. */
static int
new_session_cb(SSL* ssl, SSL_SESSION* session)
{
        struct doq_client_data* data = get_app_data(ssl);
        BIO *f;
        log_assert(data->session_file);
        verbose(1, "new session cb: the ssl session max_early_data_size is %u",
                (unsigned)SSL_SESSION_get_max_early_data(session));
        f = BIO_new_file(data->session_file, "w");
        if(!f) {
                log_err("Could not open %s: %s", data->session_file,
                        strerror(errno));
                return 0;
        }
        PEM_write_bio_SSL_SESSION(f, session);
        BIO_free(f);
        verbose(1, "written tls session to %s", data->session_file);
        return 0;
}

/** setup the TLS context */
static SSL_CTX*
ctx_client_setup(void)
{
        SSL_CTX* ctx = SSL_CTX_new(TLS_client_method());
        if(!ctx) {
                log_crypto_err("Could not SSL_CTX_new");
                exit(1);
        }
        SSL_CTX_set_min_proto_version(ctx, TLS1_3_VERSION);
        SSL_CTX_set_max_proto_version(ctx, TLS1_3_VERSION);
        SSL_CTX_set_default_verify_paths(ctx);
#ifdef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
        if(ngtcp2_crypto_quictls_configure_client_context(ctx) != 0) {
                log_err("ngtcp2_crypto_quictls_configure_client_context failed");
                exit(1);
        }
#elif defined(MAKE_QUIC_METHOD)
        memset(&quic_method, 0, sizeof(quic_method));
        quic_method.set_encryption_secrets = &set_encryption_secrets;
        quic_method.add_handshake_data = &add_handshake_data;
        quic_method.flush_flight = &flush_flight;
        quic_method.send_alert = &send_alert;
        SSL_CTX_set_quic_method(ctx, &quic_method);
#endif
        return ctx;
}


/* setup the TLS object */
static SSL*
ssl_client_setup(struct doq_client_data* data)
{
#ifdef USE_NGTCP2_CRYPTO_OSSL
        int ret;
#endif
        SSL* ssl = SSL_new(data->ctx);
        if(!ssl) {
                log_crypto_err("Could not SSL_new");
                exit(1);
        }
#ifdef USE_NGTCP2_CRYPTO_OSSL
        if((ret=ngtcp2_crypto_ossl_ctx_new(&data->ossl_ctx, NULL)) != 0) {
                log_err("ngtcp2_crypto_ossl_ctx_new failed: %s",
                        ngtcp2_strerror(ret));
                exit(1);
        }
        ngtcp2_crypto_ossl_ctx_set_ssl(data->ossl_ctx, ssl);
        if(ngtcp2_crypto_ossl_configure_client_session(ssl) != 0) {
                log_err("ngtcp2_crypto_ossl_configure_client_session failed");
                exit(1);
        }
#endif
        set_app_data(ssl, data);
        SSL_set_connect_state(ssl);
        if(!SSL_set_fd(ssl, data->fd)) {
                log_crypto_err("Could not SSL_set_fd");
                exit(1);
        }
#ifndef USE_NGTCP2_CRYPTO_OSSL
        if((data->quic_version & 0xff000000) == 0xff000000) {
                SSL_set_quic_use_legacy_codepoint(ssl, 1);
        } else {
                SSL_set_quic_use_legacy_codepoint(ssl, 0);
        }
#endif
        SSL_set_alpn_protos(ssl, (const unsigned char *)"\x03""doq", 4);
        /* send the SNI host name */
        SSL_set_tlsext_host_name(ssl, "localhost");
        return ssl;
}

/** get packet ecn information */
static uint32_t
msghdr_get_ecn(struct msghdr* msg, int family)
{
#ifndef S_SPLINT_S
        struct cmsghdr* cmsg;
        if(family == AF_INET6) {
                for(cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
                        cmsg = CMSG_NXTHDR(msg, cmsg)) {
                        if(cmsg->cmsg_level == IPPROTO_IPV6 &&
                                cmsg->cmsg_type == IPV6_TCLASS &&
                                cmsg->cmsg_len != 0) {
                                uint8_t* ecn = (uint8_t*)CMSG_DATA(cmsg);
                                return *ecn;
                        }
                }
                return 0;
        }
        for(cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
                cmsg = CMSG_NXTHDR(msg, cmsg)) {
                if(cmsg->cmsg_level == IPPROTO_IP &&
                        cmsg->cmsg_type == IP_TOS &&
                        cmsg->cmsg_len != 0) {
                        uint8_t* ecn = (uint8_t*)CMSG_DATA(cmsg);
                        return *ecn;
                }
        }
        return 0;
#endif /* S_SPLINT_S */
}

/** set the ecn on the transmission */
static void
set_ecn(int fd, int family, uint32_t ecn)
{
        unsigned int val = ecn;
        if(family == AF_INET6) {
                if(setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &val,
                        (socklen_t)sizeof(val)) == -1) {
                        log_err("setsockopt(.. IPV6_TCLASS ..): %s",
                                strerror(errno));
                }
                return;
        }
        if(setsockopt(fd, IPPROTO_IP, IP_TOS, &val,
                (socklen_t)sizeof(val)) == -1) {
                log_err("setsockopt(.. IP_TOS ..): %s",
                        strerror(errno));
        }
}

/** send a packet */
static int
doq_client_send_pkt(struct doq_client_data* data, uint32_t ecn, uint8_t* buf,
        size_t buf_len, int is_blocked_pkt, int* send_is_blocked)
{
        struct msghdr msg;
        struct iovec iov[1];
        ssize_t ret;
        iov[0].iov_base = buf;
        iov[0].iov_len = buf_len;
        memset(&msg, 0, sizeof(msg));
        msg.msg_name = (void*)&data->dest_addr;
        msg.msg_namelen = data->dest_addr_len;
        msg.msg_iov = iov;
        msg.msg_iovlen = 1;
        set_ecn(data->fd, data->dest_addr.ss_family, ecn);

        for(;;) {
                ret = sendmsg(data->fd, &msg, MSG_DONTWAIT);
                if(ret == -1 && errno == EINTR)
                        continue;
                break;
        }
        if(ret == -1) {
                if(errno == EAGAIN) {
                        if(buf_len >
                                sldns_buffer_capacity(data->blocked_pkt))
                                return 0; /* Cannot store it, but the buffers
                                are equal length and large enough, so this
                                should not happen. */
                        data->have_blocked_pkt = 1;
                        if(send_is_blocked)
                                *send_is_blocked = 1;
                        /* If we already send the previously blocked packet,
                         * no need to copy it, otherwise store the packet for
                         * later. */
                        if(!is_blocked_pkt) {
                                data->blocked_pkt_pi.ecn = ecn;
                                sldns_buffer_clear(data->blocked_pkt);
                                sldns_buffer_write(data->blocked_pkt, buf,
                                        buf_len);
                                sldns_buffer_flip(data->blocked_pkt);
                        }
                        return 0;
                }
                log_err("doq sendmsg: %s", strerror(errno));
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                ngtcp2_ccerr_set_application_error(&data->ccerr, -1, NULL, 0);
#else
                ngtcp2_connection_close_error_set_application_error(&data->last_error, -1, NULL, 0);
#endif
                return 0;
        }
        return 1;
}

/** change event write on fd to when we have data or when congested */
static void
event_change_write(struct doq_client_data* data, int do_write)
{
        ub_event_del(data->ev);
        if(do_write) {
                ub_event_add_bits(data->ev, UB_EV_WRITE);
        } else {
                ub_event_del_bits(data->ev, UB_EV_WRITE);
        }
        if(ub_event_add(data->ev, NULL) != 0) {
                fatal_exit("could not ub_event_add");
        }
}

/** write the connection close, with possible error */
static void
write_conn_close(struct doq_client_data* data)
{
        struct ngtcp2_path_storage ps;
        struct ngtcp2_pkt_info pi;
        ngtcp2_ssize ret;
        if(!data->conn ||
#ifdef HAVE_NGTCP2_CONN_IN_CLOSING_PERIOD
                ngtcp2_conn_in_closing_period(data->conn) ||
#else
                ngtcp2_conn_is_in_closing_period(data->conn) ||
#endif
#ifdef HAVE_NGTCP2_CONN_IN_DRAINING_PERIOD
                ngtcp2_conn_in_draining_period(data->conn)
#else
                ngtcp2_conn_is_in_draining_period(data->conn)
#endif
                )
                return;
        /* Drop blocked packet if there is one, the connection is being
         * closed. And thus no further data traffic. */
        data->have_blocked_pkt = 0;
        if(
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                data->ccerr.type == NGTCP2_CCERR_TYPE_IDLE_CLOSE
#else
                data->last_error.type ==
                NGTCP2_CONNECTION_CLOSE_ERROR_CODE_TYPE_TRANSPORT_IDLE_CLOSE
#endif
                ) {
                /* do not call ngtcp2_conn_write_connection_close on the
                 * connection because the ngtcp2_conn_handle_expiry call
                 * has returned NGTCP2_ERR_IDLE_CLOSE. But continue to close
                 * the connection. */
                return;
        }
        verbose(1, "write connection close");
        ngtcp2_path_storage_zero(&ps);
        sldns_buffer_clear(data->pkt_buf);
        ret = ngtcp2_conn_write_connection_close(
                data->conn, &ps.path, &pi, sldns_buffer_begin(data->pkt_buf),
                sldns_buffer_remaining(data->pkt_buf),
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                &data->ccerr
#else
                &data->last_error
#endif
                , get_timestamp_nanosec());
        if(ret < 0) {
                log_err("ngtcp2_conn_write_connection_close failed: %s",
                        ngtcp2_strerror(ret));
                return;
        }
        verbose(1, "write connection close packet length %d", (int)ret);
        if(ret == 0)
                return;
        doq_client_send_pkt(data, pi.ecn, sldns_buffer_begin(data->pkt_buf),
                ret, 0, NULL);
}

/** disconnect we are done */
static void
disconnect(struct doq_client_data* data)
{
        verbose(1, "disconnect");
        write_conn_close(data);
        ub_event_base_loopexit(data->base);
}

/** the expire timer callback */
void doq_client_timer_cb(int ATTR_UNUSED(fd),
        short ATTR_UNUSED(bits), void* arg)
{
        struct doq_client_data* data = (struct doq_client_data*)arg;
        ngtcp2_tstamp now = get_timestamp_nanosec();
        int rv;

        verbose(1, "doq expire_timer");
        data->expire_timer_added = 0;
        rv = ngtcp2_conn_handle_expiry(data->conn, now);
        if(rv != 0) {
                log_err("ngtcp2_conn_handle_expiry failed: %s",
                        ngtcp2_strerror(rv));
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                ngtcp2_ccerr_set_liberr(&data->ccerr, rv, NULL, 0);
#else
                ngtcp2_connection_close_error_set_transport_error_liberr(
                        &data->last_error, rv, NULL, 0);
#endif
                disconnect(data);
                return;
        }
        update_timer(data);
        on_write(data);
}

/** update the timers */
static void
update_timer(struct doq_client_data* data)
{
        ngtcp2_tstamp expiry = ngtcp2_conn_get_expiry(data->conn);
        ngtcp2_tstamp now = get_timestamp_nanosec();
        ngtcp2_tstamp t;
        struct timeval tv;

        if(expiry <= now) {
                /* the timer has already expired, add with zero timeout */
                t = 0;
        } else {
                t = expiry - now;
        }

        /* set the timer */
        if(data->expire_timer_added) {
                ub_timer_del(data->expire_timer);
                data->expire_timer_added = 0;
        }
        memset(&tv, 0, sizeof(tv));
        tv.tv_sec = t / NGTCP2_SECONDS;
        tv.tv_usec = (t / NGTCP2_MICROSECONDS)%1000000;
        verbose(1, "update_timer in %d.%6.6d secs", (int)tv.tv_sec,
                (int)tv.tv_usec);
        if(ub_timer_add(data->expire_timer, data->base,
                &doq_client_timer_cb, data, &tv) != 0) {
                log_err("timer_add failed: could not add expire timer");
                return;
        }
        data->expire_timer_added = 1;
}

/** perform read operations on fd */
static void
on_read(struct doq_client_data* data)
{
        struct sockaddr_storage addr;
        struct iovec iov[1];
        struct msghdr msg;
        union {
                struct cmsghdr hdr;
                char buf[256];
        } ancil;
        int i;
        ssize_t rcv;
        ngtcp2_pkt_info pi;
        int rv;
        struct ngtcp2_path path;

        for(i=0; i<10; i++) {
                msg.msg_name = &addr;
                msg.msg_namelen = (socklen_t)sizeof(addr);
                iov[0].iov_base = sldns_buffer_begin(data->pkt_buf);
                iov[0].iov_len = sldns_buffer_remaining(data->pkt_buf);
                msg.msg_iov = iov;
                msg.msg_iovlen = 1;
                msg.msg_control = ancil.buf;
#ifndef S_SPLINT_S
                msg.msg_controllen = sizeof(ancil.buf);
#endif /* S_SPLINT_S */
                msg.msg_flags = 0;

                rcv = recvmsg(data->fd, &msg, MSG_DONTWAIT);
                if(rcv == -1) {
                        if(errno == EINTR || errno == EAGAIN)
                                break;
                        log_err_addr("doq recvmsg", strerror(errno),
                                &data->dest_addr, sizeof(data->dest_addr_len));
                        break;
                }

                pi.ecn = msghdr_get_ecn(&msg, addr.ss_family);
                verbose(1, "recvmsg %d ecn=0x%x", (int)rcv, (int)pi.ecn);

                memset(&path, 0, sizeof(path));
                path.local.addr = (void*)&data->local_addr;
                path.local.addrlen = data->local_addr_len;
                path.remote.addr = (void*)msg.msg_name;
                path.remote.addrlen = msg.msg_namelen;
                rv = ngtcp2_conn_read_pkt(data->conn, &path, &pi,
                        iov[0].iov_base, rcv, get_timestamp_nanosec());
                if(rv != 0) {
                        log_err("ngtcp2_conn_read_pkt failed: %s",
                                ngtcp2_strerror(rv));
                        if(
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                                data->ccerr.error_code == 0
#else
                                data->last_error.error_code == 0
#endif
                                ) {
                                if(rv == NGTCP2_ERR_CRYPTO) {
                                        /* in picotls the tls alert may need
                                         * to be copied, but this is with
                                         * openssl. And we have the value
                                         * data.tls_alert. */
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                                        ngtcp2_ccerr_set_tls_alert(
                                                &data->ccerr, data->tls_alert,
                                                NULL, 0);
#else
                                        ngtcp2_connection_close_error_set_transport_error_tls_alert(
                                                &data->last_error,
                                                data->tls_alert, NULL, 0);
#endif
                                } else {
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                                        ngtcp2_ccerr_set_liberr(&data->ccerr,
                                                rv, NULL, 0);
#else
                                        ngtcp2_connection_close_error_set_transport_error_liberr(
                                                &data->last_error, rv, NULL,
                                                0);
#endif
                                }
                        }
                        disconnect(data);
                        return;
                }
        }

        update_timer(data);
}

/** the write of this query has completed, it has spooled to packets,
 * set it to have the write done and move it to the list of receive streams. */
static void
query_write_is_done(struct doq_client_data* data,
        struct doq_client_stream* str)
{
        if(verbosity > 0) {
                char* logs = client_stream_string(str);
                verbose(1, "query %s write is done", logs);
                free(logs);
        }
        str->write_is_done = 1;
        stream_list_move(str, data->query_list_send, data->query_list_receive);
}

/** write the data streams, if possible */
static int
write_streams(struct doq_client_data* data)
{
        ngtcp2_path_storage ps;
        ngtcp2_tstamp ts = get_timestamp_nanosec();
        struct doq_client_stream* str, *next;
        uint32_t flags;
        /* number of bytes that can be sent without packet pacing */
        size_t send_quantum = ngtcp2_conn_get_send_quantum(data->conn);
        /* Overhead is the stream overhead of adding a header onto the data,
         * this make sure the number of bytes to send in data bytes plus
         * the overhead overshoots the target quantum by a smaller margin,
         * and then it stops sending more bytes. With zero it would overshoot
         * more, an accurate number would not overshoot. It is based on the
         * stream frame header size. */
        size_t accumulated_send = 0, overhead_stream = 24, overhead_pkt = 60,
                max_packet_size = 1200;
        size_t num_packets = 0, max_packets = 65535;
        ngtcp2_path_storage_zero(&ps);
        str = data->query_list_send->first;

        if(data->cc_algo != NGTCP2_CC_ALGO_BBR
#ifdef NGTCP2_CC_ALGO_BBR_V2
                && data->cc_algo != NGTCP2_CC_ALGO_BBR_V2
#endif
#ifdef NGTCP2_CC_ALGO_BBR2
                && data->cc_algo != NGTCP2_CC_ALGO_BBR2
#endif
                ) {
                /* If we do not have a packet pacing congestion control
                 * algorithm, limit the number of packets. */
                max_packets = 10;
        }

        /* loop like this, because at the start, the send list is empty,
         * and we want to send handshake packets. But when there is a
         * send_list, loop through that. */
        for(;;) {
                int64_t stream_id;
                ngtcp2_pkt_info pi;
                ngtcp2_vec datav[2];
                size_t datav_count = 0;
                int fin;
                ngtcp2_ssize ret;
                ngtcp2_ssize ndatalen = 0;
                int send_is_blocked = 0;

                if(str) {
                        /* pick up next in case this one is deleted */
                        next = str->next;
                        if(verbosity > 0) {
                                char* logs = client_stream_string(str);
                                verbose(1, "query %s write stream", logs);
                                free(logs);
                        }
                        stream_id = str->stream_id;
                        fin = 1;
                        if(str->nwrite < 2) {
                                str->data_tcplen = htons(str->data_len);
                                datav[0].base = ((uint8_t*)&str->data_tcplen)+str->nwrite;
                                datav[0].len = 2-str->nwrite;
                                datav[1].base = str->data;
                                datav[1].len = str->data_len;
                                datav_count = 2;
                        } else {
                                datav[0].base = str->data + (str->nwrite-2);
                                datav[0].len = str->data_len - (str->nwrite-2);
                                datav_count = 1;
                        }
                } else {
                        next = NULL;
                        verbose(1, "write stream -1.");
                        stream_id = -1;
                        fin = 0;
                        datav[0].base = NULL;
                        datav[0].len = 0;
                        datav_count = 1;
                }

                /* Does the first data entry fit into the send quantum? */
                /* Check if the data size sent, with a max of one full packet,
                 * with added stream header and packet header is allowed
                 * within the send quantum number of bytes. If not, it does
                 * not fit, and wait. */
                if(accumulated_send == 0 && ((datav_count == 1 &&
                        (datav[0].len>max_packet_size?max_packet_size:
                        datav[0].len)+overhead_stream+overhead_pkt >
                        send_quantum) ||
                        (datav_count == 2 &&
                        (datav[0].len+datav[1].len>max_packet_size?
                        max_packet_size:datav[0].len+datav[1].len)
                        +overhead_stream+overhead_pkt > send_quantum))) {
                        /* congestion limited */
                        ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
                        event_change_write(data, 0);
                        /* update the timer to wait until it is possible to
                         * write again */
                        update_timer(data);
                        return 0;
                }
                flags = 0;
                if(str && str->next != NULL) {
                        /* Coalesce more data from more streams into this
                         * packet, if possible */
                        /* There is more than one data entry in this send
                         * quantum, does the next one fit in the quantum? */
                        size_t this_send, possible_next_send;
                        if(datav_count == 1)
                                this_send = datav[0].len;
                        else    this_send = datav[0].len + datav[1].len;
                        if(this_send > max_packet_size)
                                this_send = max_packet_size;
                        if(str->next->nwrite < 2)
                                possible_next_send = (2-str->next->nwrite) +
                                        str->next->data_len;
                        else    possible_next_send = str->next->data_len -
                                        (str->next->nwrite - 2);
                        if(possible_next_send > max_packet_size)
                                possible_next_send = max_packet_size;
                        /* Check if the data lengths that writev returned
                         * with stream headers added up so far, in
                         * accumulated_send, with added the data length
                         * of this send, with a max of one full packet, and
                         * the data length of the next possible send, with
                         * a max of one full packet, with a stream header for
                         * this_send and a stream header for the next possible
                         * send and a packet header, fit in the send quantum
                         * number of bytes. If so, ask to add more content
                         * to the packet with the more flag. */
                        if(accumulated_send + this_send + possible_next_send
                                +2*overhead_stream+ overhead_pkt < send_quantum)
                                flags |= NGTCP2_WRITE_STREAM_FLAG_MORE;
                }
                if(fin) {
                        /* This is the final part of data for this stream */
                        flags |= NGTCP2_WRITE_STREAM_FLAG_FIN;
                }
                sldns_buffer_clear(data->pkt_buf);
                ret = ngtcp2_conn_writev_stream(data->conn, &ps.path, &pi,
                        sldns_buffer_begin(data->pkt_buf),
                        sldns_buffer_remaining(data->pkt_buf), &ndatalen,
                        flags, stream_id, datav, datav_count, ts);
                if(ret < 0) {
                        if(ret == NGTCP2_ERR_WRITE_MORE) {
                                if(str) {
                                        str->nwrite += ndatalen;
                                        if(str->nwrite >= str->data_len+2)
                                                query_write_is_done(data, str);
                                        str = next;
                                        accumulated_send += ndatalen + overhead_stream;
                                        continue;
                                }
                        }
                        log_err("ngtcp2_conn_writev_stream failed: %s",
                                ngtcp2_strerror(ret));
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
                        ngtcp2_ccerr_set_liberr(&data->ccerr, ret, NULL, 0);
#else
                        ngtcp2_connection_close_error_set_transport_error_liberr(
                                &data->last_error, ret, NULL, 0);
#endif
                        disconnect(data);
                        return 0;
                }
                verbose(1, "writev_stream pkt size %d ndatawritten %d",
                        (int)ret, (int)ndatalen);
                if(ndatalen >= 0 && str) {
                        /* add the new write offset */
                        str->nwrite += ndatalen;
                        if(str->nwrite >= str->data_len+2)
                                query_write_is_done(data, str);
                }
                if(ret == 0) {
                        /* congestion limited */
                        ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
                        event_change_write(data, 0);
                        /* update the timer to wait until it is possible to
                         * write again */
                        update_timer(data);
                        return 0;
                }
                if(!doq_client_send_pkt(data, pi.ecn,
                        sldns_buffer_begin(data->pkt_buf), ret, 0,
                        &send_is_blocked)) {
                        if(send_is_blocked) {
                                /* Blocked packet, wait until it is possible
                                 * to write again and also set a timer. */
                                event_change_write(data, 1);
                                update_timer(data);
                                return 0;
                        }
                        /* Packet could not be sent. Like lost and timeout. */
                        ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
                        event_change_write(data, 0);
                        update_timer(data);
                        return 0;
                }
                /* continue */
                if((size_t)ret >= send_quantum)
                        break;
                send_quantum -= ret;
                accumulated_send = 0;
                str = next;
                if(str == NULL)
                        break;
                if(++num_packets == max_packets)
                        break;
        }
        ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
        event_change_write(data, 1);
        return 1;
}

/** send the blocked packet now that the stream is writable again. */
static int
send_blocked_pkt(struct doq_client_data* data)
{
        ngtcp2_tstamp ts = get_timestamp_nanosec();
        int send_is_blocked = 0;
        if(!doq_client_send_pkt(data, data->blocked_pkt_pi.ecn,
                sldns_buffer_begin(data->pkt_buf),
                sldns_buffer_limit(data->pkt_buf), 1, &send_is_blocked)) {
                if(send_is_blocked) {
                        /* Send was blocked, again. Wait, again to retry. */
                        event_change_write(data, 1);
                        /* make sure the timer is set while waiting */
                        update_timer(data);
                        return 0;
                }
                /* The packed could not be sent. Like it was lost, timeout. */
                data->have_blocked_pkt = 0;
                ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
                event_change_write(data, 0);
                update_timer(data);
                return 0;
        }
        /* The blocked packet has been sent, the holding buffer can be
         * cleared. */
        data->have_blocked_pkt = 0;
        ngtcp2_conn_update_pkt_tx_time(data->conn, ts);
        return 1;
}

/** perform write operations, if any, on fd */
static void
on_write(struct doq_client_data* data)
{
        if(data->have_blocked_pkt) {
                if(!send_blocked_pkt(data))
                        return;
        }
        if(
#ifdef HAVE_NGTCP2_CONN_IN_CLOSING_PERIOD
                ngtcp2_conn_in_closing_period(data->conn)
#else
                ngtcp2_conn_is_in_closing_period(data->conn)
#endif
                )
                return;
        if(!write_streams(data))
                return;
        update_timer(data);
}

/** callback for main listening file descriptor */
void
doq_client_event_cb(int ATTR_UNUSED(fd), short bits, void* arg)
{
        struct doq_client_data* data = (struct doq_client_data*)arg;
        verbose(1, "doq_client_event_cb %s%s%s",
                ((bits&UB_EV_READ)!=0?"EV_READ":""),
                ((bits&(UB_EV_READ|UB_EV_WRITE))==(UB_EV_READ|UB_EV_WRITE)?
                " ":""),
                ((bits&UB_EV_WRITE)!=0?"EV_WRITE":""));
        if((bits&UB_EV_READ)) {
                on_read(data);
        }
        /* Perform the write operation anyway. The read operation may
         * have produced data, or there is content waiting and it is possible
         * to write that. */
        on_write(data);
}

/** read the TLS session from file */
static int
early_data_setup_session(struct doq_client_data* data)
{
        SSL_SESSION* session;
        BIO* f = BIO_new_file(data->session_file, "r");
        if(f == NULL) {
                if(errno == ENOENT) {
                        verbose(1, "session file %s does not exist",
                                data->session_file);
                        return 0;
                }
                log_err("Could not read %s: %s", data->session_file,
                        strerror(errno));
                return 0;
        }
        session = PEM_read_bio_SSL_SESSION(f, NULL, 0, NULL);
        if(session == NULL) {
                log_crypto_err("Could not read session file with PEM_read_bio_SSL_SESSION");
                BIO_free(f);
                return 0;
        }
        BIO_free(f);
        if(!SSL_set_session(data->ssl, session)) {
                log_crypto_err("Could not SSL_set_session");
                SSL_SESSION_free(session);
                return 0;
        }
        if(SSL_SESSION_get_max_early_data(session) == 0) {
                log_err("TLS session early data is 0");
                SSL_SESSION_free(session);
                return 0;
        }
#ifdef USE_NGTCP2_CRYPTO_OSSL
        SSL_set_quic_tls_early_data_enabled(data->ssl, 1);
#else
        SSL_set_quic_early_data_enabled(data->ssl, 1);
#endif
        SSL_SESSION_free(session);
        return 1;
}

#ifndef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
/** parse one line from the transport file */
static int
transport_parse_line(struct ngtcp2_transport_params* params, char* line)
{
        if(strncmp(line, "initial_max_streams_bidi=", 25) == 0) {
                params->initial_max_streams_bidi = atoi(line+25);
                return 1;
        }
        if(strncmp(line, "initial_max_streams_uni=", 24) == 0) {
                params->initial_max_streams_uni = atoi(line+24);
                return 1;
        }
        if(strncmp(line, "initial_max_stream_data_bidi_local=", 35) == 0) {
                params->initial_max_stream_data_bidi_local = atoi(line+35);
                return 1;
        }
        if(strncmp(line, "initial_max_stream_data_bidi_remote=", 36) == 0) {
                params->initial_max_stream_data_bidi_remote = atoi(line+36);
                return 1;
        }
        if(strncmp(line, "initial_max_stream_data_uni=", 28) == 0) {
                params->initial_max_stream_data_uni = atoi(line+28);
                return 1;
        }
        if(strncmp(line, "initial_max_data=", 17) == 0) {
                params->initial_max_data = atoi(line+17);
                return 1;
        }
        if(strncmp(line, "active_connection_id_limit=", 27) == 0) {
                params->active_connection_id_limit = atoi(line+27);
                return 1;
        }
        if(strncmp(line, "max_datagram_frame_size=", 24) == 0) {
                params->max_datagram_frame_size = atoi(line+24);
                return 1;
        }
        return 0;
}
#endif /* HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS */

/** setup the early data transport file and read it */
static int
early_data_setup_transport(struct doq_client_data* data)
{
#ifdef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
        FILE* in;
        uint8_t buf[1024];
        size_t len;
        int rv;
        in = fopen(data->transport_file, "r");
        if(!in) {
                if(errno == ENOENT) {
                        verbose(1, "transport file %s does not exist",
                                data->transport_file);
                        return 0;
                }
                perror(data->transport_file);
                return 0;
        }
        len = fread(buf, 1, sizeof(buf), in);
        if(ferror(in)) {
                log_err("%s: read failed: %s", data->transport_file,
                        strerror(errno));
                fclose(in);
                return 0;
        }
        fclose(in);
        rv = ngtcp2_conn_decode_and_set_0rtt_transport_params(data->conn,
                buf, len);
        if(rv != 0) {
                log_err("ngtcp2_conn_decode_and_set_0rtt_transport_params failed: %s",
                        ngtcp2_strerror(rv));
                return 0;
        }
        return 1;
#else
        FILE* in;
        char buf[1024];
        struct ngtcp2_transport_params params;
        memset(&params, 0, sizeof(params));
        in = fopen(data->transport_file, "r");
        if(!in) {
                if(errno == ENOENT) {
                        verbose(1, "transport file %s does not exist",
                                data->transport_file);
                        return 0;
                }
                perror(data->transport_file);
                return 0;
        }
        while(!feof(in)) {
                if(!fgets(buf, sizeof(buf), in)) {
                        log_err("%s: read failed: %s", data->transport_file,
                                strerror(errno));
                        fclose(in);
                        return 0;
                }
                if(!transport_parse_line(&params, buf)) {
                        log_err("%s: could not parse line '%s'",
                                data->transport_file, buf);
                        fclose(in);
                        return 0;
                }
        }
        fclose(in);
        ngtcp2_conn_set_early_remote_transport_params(data->conn, &params);
#endif
        return 1;
}

/** setup for early data, read the transport file and session file */
static void
early_data_setup(struct doq_client_data* data)
{
        if(!early_data_setup_session(data)) {
                verbose(1, "TLS session resumption failed, early data is disabled");
                data->early_data_enabled = 0;
                return;
        }
        if(!early_data_setup_transport(data)) {
                verbose(1, "Transport parameters set failed, early data is disabled");
                data->early_data_enabled = 0;
                return;
        }
}

/** start the early data transmission */
static void
early_data_start(struct doq_client_data* data)
{
        query_streams_start(data);
        on_write(data);
}

/** create doq_client_data */
static struct doq_client_data*
create_doq_client_data(const char* svr, int port, struct ub_event_base* base,
        const char* transport_file, const char* session_file, int quiet)
{
        struct doq_client_data* data;
        data = calloc(1, sizeof(*data));
        if(!data) fatal_exit("calloc failed: out of memory");
        data->base = base;
#ifdef USE_NGTCP2_CRYPTO_OSSL
        /* Initialize the ossl crypto, it is harmless to call twice,
         * and this is before use of doq connections. */
        if(ngtcp2_crypto_ossl_init() != 0)
                fatal_exit("ngtcp2_crypto_oss_init failed");
#elif defined(HAVE_NGTCP2_CRYPTO_QUICTLS_INIT)
        if(ngtcp2_crypto_quictls_init() != 0)
                fatal_exit("ngtcp2_crypto_quictls_init failed");
#endif
        data->rnd = ub_initstate(NULL);
        if(!data->rnd) fatal_exit("ub_initstate failed: out of memory");
        data->svr = svr;
        get_dest_addr(data, svr, port);
        data->port = port;
        data->quiet = quiet;
        data->pkt_buf = sldns_buffer_new(65552);
        if(!data->pkt_buf)
                fatal_exit("sldns_buffer_new failed: out of memory");
        data->blocked_pkt = sldns_buffer_new(65552);
        if(!data->blocked_pkt)
                fatal_exit("sldns_buffer_new failed: out of memory");
        data->fd = open_svr_udp(data);
        get_local_addr(data);
        data->conn = conn_client_setup(data);
#ifdef HAVE_NGTCP2_CCERR_DEFAULT
        ngtcp2_ccerr_default(&data->ccerr);
#else
        ngtcp2_connection_close_error_default(&data->last_error);
#endif
        data->transport_file = transport_file;
        data->session_file = session_file;
        if(data->transport_file && data->session_file)
                data->early_data_enabled = 1;

        generate_static_secret(data, 32);
        data->ctx = ctx_client_setup();
        if(data->session_file) {
                SSL_CTX_set_session_cache_mode(data->ctx,
                        SSL_SESS_CACHE_CLIENT |
                        SSL_SESS_CACHE_NO_INTERNAL_STORE);
                SSL_CTX_sess_set_new_cb(data->ctx, new_session_cb);
        }
        data->ssl = ssl_client_setup(data);
#ifdef USE_NGTCP2_CRYPTO_OSSL
        ngtcp2_conn_set_tls_native_handle(data->conn, data->ossl_ctx);
#else
        ngtcp2_conn_set_tls_native_handle(data->conn, data->ssl);
#endif
        if(data->early_data_enabled)
                early_data_setup(data);

        data->ev = ub_event_new(base, data->fd, UB_EV_READ | UB_EV_WRITE |
                UB_EV_PERSIST, doq_client_event_cb, data);
        if(!data->ev) {
                fatal_exit("could not ub_event_new");
        }
        if(ub_event_add(data->ev, NULL) != 0) {
                fatal_exit("could not ub_event_add");
        }
        data->expire_timer = ub_event_new(data->base, -1,
                UB_EV_TIMEOUT, &doq_client_timer_cb, data);
        if(!data->expire_timer)
                fatal_exit("could not ub_event_new");
        data->query_list_start = stream_list_create();
        data->query_list_send = stream_list_create();
        data->query_list_receive = stream_list_create();
        data->query_list_stop = stream_list_create();
        return data;
}

/** delete doq_client_data */
static void
delete_doq_client_data(struct doq_client_data* data)
{
        if(!data)
                return;
#if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
        if(data->conn && data->dest_addr_len != 0) {
                if(addr_is_ip6(&data->dest_addr, data->dest_addr_len)) {
#  if defined(NGTCP2_USE_GENERIC_SOCKADDR) || defined(NGTCP2_USE_GENERIC_IPV6_SOCKADDR)
                        const struct ngtcp2_path* path6 = ngtcp2_conn_get_path(data->conn);
                        free(path6->local.addr);
                        free(path6->remote.addr);
#  endif
                } else {
#  if defined(NGTCP2_USE_GENERIC_SOCKADDR)
                        const struct ngtcp2_path* path = ngtcp2_conn_get_path(data->conn);
                        free(path->local.addr);
                        free(path->remote.addr);
#  endif
                }
        }
#endif
        /* Remove the app data from ngtcp2 before SSL_free of conn->ssl,
         * because the ngtcp2 conn is deleted. */
        SSL_set_app_data(data->ssl, NULL);
        SSL_free(data->ssl);
#ifdef USE_NGTCP2_CRYPTO_OSSL
        ngtcp2_crypto_ossl_ctx_del(data->ossl_ctx);
#endif
        ngtcp2_conn_del(data->conn);
        sldns_buffer_free(data->pkt_buf);
        sldns_buffer_free(data->blocked_pkt);
        if(data->fd != -1)
                sock_close(data->fd);
        SSL_CTX_free(data->ctx);
        stream_list_free(data->query_list_start);
        stream_list_free(data->query_list_send);
        stream_list_free(data->query_list_receive);
        stream_list_free(data->query_list_stop);
        ub_randfree(data->rnd);
        if(data->ev) {
                ub_event_del(data->ev);
                ub_event_free(data->ev);
        }
        if(data->expire_timer_added)
                ub_timer_del(data->expire_timer);
        ub_event_free(data->expire_timer);
        free(data->static_secret_data);
        free(data);
}

/** create the event base that registers events and timers */
static struct ub_event_base*
create_event_base(time_t* secs, struct timeval* now)
{
        struct ub_event_base* base;
        const char *evnm="event", *evsys="", *evmethod="";

        memset(now, 0, sizeof(*now));
        base = ub_default_event_base(1, secs, now);
        if(!base) fatal_exit("could not create ub_event base");

        ub_get_event_sys(base, &evnm, &evsys, &evmethod);
        if(verbosity) log_info("%s %s uses %s method", evnm, evsys, evmethod);

        return base;
}

/** enter a query into the query list */
static void
client_enter_query_buf(struct doq_client_data* data, struct sldns_buffer* buf)
{
        struct doq_client_stream* str;
        str = client_stream_create(buf);
        if(!str)
                fatal_exit("client_stream_create failed: out of memory");
        stream_list_append(data->query_list_start, str);
}

/** enter the queries into the query list */
static void
client_enter_queries(struct doq_client_data* data, char** qs, int count)
{
        int i;
        for(i=0; i<count; i+=3) {
                struct sldns_buffer* buf = NULL;
                buf = make_query(qs[i], qs[i+1], qs[i+2]);
                if(verbosity > 0) {
                        char* str;
                        log_buf(1, "send query", buf);
                        str = sldns_wire2str_pkt(sldns_buffer_begin(buf),
                                sldns_buffer_limit(buf));
                        if(!str) verbose(1, "could not sldns_wire2str_pkt");
                        else verbose(1, "send query:\n%s", str);
                        free(str);
                }
                client_enter_query_buf(data, buf);
                sldns_buffer_free(buf);
        }
}

/** run the dohclient queries */
static void run(const char* svr, int port, char** qs, int count,
        const char* transport_file, const char* session_file, int quiet)
{
        time_t secs = 0;
        struct timeval now;
        struct ub_event_base* base;
        struct doq_client_data* data;

        /* setup */
        base = create_event_base(&secs, &now);
        data = create_doq_client_data(svr, port, base, transport_file,
                session_file, quiet);
        client_enter_queries(data, qs, count);
        if(data->early_data_enabled)
                early_data_start(data);

        /* run the queries */
        ub_event_base_dispatch(base);

        /* cleanup */
        delete_doq_client_data(data);
        ub_event_base_free(base);
}
#endif /* HAVE_NGTCP2 */

#ifdef HAVE_NGTCP2
/** getopt global, in case header files fail to declare it. */
extern int optind;
/** getopt global, in case header files fail to declare it. */
extern char* optarg;
int main(int ATTR_UNUSED(argc), char** ATTR_UNUSED(argv))
{
        int c;
        int port = UNBOUND_DNS_OVER_QUIC_PORT, quiet = 0;
        const char* svr = "127.0.0.1", *transport_file = NULL,
                *session_file = NULL;
#ifdef USE_WINSOCK
        WSADATA wsa_data;
        if(WSAStartup(MAKEWORD(2,2), &wsa_data) != 0) {
                printf("WSAStartup failed\n");
                return 1;
        }
#endif
        checklock_set_output_name("ublocktrace-doqclient");
        checklock_start();
        log_init(0, 0, 0);
        log_ident_set("doqclient");

        while((c=getopt(argc, argv, "hp:qs:vx:y:")) != -1) {
                switch(c) {
                        case 'p':
                                if(atoi(optarg)==0 && strcmp(optarg,"0")!=0) {
                                        printf("error parsing port, "
                                            "number expected: %s\n", optarg);
                                        return 1;
                                }
                                port = atoi(optarg);
                                break;
                        case 'q':
                                quiet++;
                                break;
                        case 's':
                                svr = optarg;
                                break;
                        case 'v':
                                verbosity++;
                                break;
                        case 'x':
                                transport_file = optarg;
                                break;
                        case 'y':
                                session_file = optarg;
                                break;
                        case 'h':
                        case '?':
                        default:
                                usage(argv);
                }
        }

        argc -= optind;
        argv += optind;

        if(argc%3!=0) {
                printf("Invalid input. Specify qname, qtype, and qclass.\n");
                return 1;
        }
        if(port == 53) {
                printf("Error: port number 53 not for DNS over QUIC. Port number 53 is not allowed to be used with DNS over QUIC. It is used for DNS datagrams.\n");
                return 1;
        }

        run(svr, port, argv, argc, transport_file, session_file, quiet);

        checklock_stop();
#ifdef USE_WINSOCK
        WSACleanup();
#endif
        return 0;
}
#else /* HAVE_NGTCP2 */
int main(int ATTR_UNUSED(argc), char** ATTR_UNUSED(argv))
{
        printf("Compiled without ngtcp2 for QUIC, cannot run doqclient.\n");
        return 1;
}
#endif /* HAVE_NGTCP2 */

/***--- definitions to make fptr_wlist work. ---***/
/* These are callbacks, similar to smallapp callbacks, except the debug
 * tool callbacks are not in it */
struct tube;
struct query_info;
#include "util/data/packed_rrset.h"
#include "daemon/worker.h"
#include "daemon/remote.h"
#include "util/fptr_wlist.h"
#include "libunbound/context.h"

void worker_handle_control_cmd(struct tube* ATTR_UNUSED(tube),
        uint8_t* ATTR_UNUSED(buffer), size_t ATTR_UNUSED(len),
        int ATTR_UNUSED(error), void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

int worker_handle_request(struct comm_point* ATTR_UNUSED(c), 
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(repinfo))
{
        log_assert(0);
        return 0;
}

int worker_handle_service_reply(struct comm_point* ATTR_UNUSED(c), 
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(reply_info))
{
        log_assert(0);
        return 0;
}

int remote_accept_callback(struct comm_point* ATTR_UNUSED(c), 
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(repinfo))
{
        log_assert(0);
        return 0;
}

int remote_control_callback(struct comm_point* ATTR_UNUSED(c), 
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(repinfo))
{
        log_assert(0);
        return 0;
}

void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

struct outbound_entry* worker_send_query(
        struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
        int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
        int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
        struct sockaddr_storage* ATTR_UNUSED(addr),
        socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
        size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
        int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
        struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
{
        log_assert(0);
        return 0;
}

#ifdef UB_ON_WINDOWS
void
worker_win_stop_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev), void* 
        ATTR_UNUSED(arg)) {
        log_assert(0);
}

void
wsvc_cron_cb(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}
#endif /* UB_ON_WINDOWS */

void 
worker_alloc_cleanup(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

struct outbound_entry* libworker_send_query(
        struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
        int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
        int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
        struct sockaddr_storage* ATTR_UNUSED(addr),
        socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
        size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
        int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
        struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
{
        log_assert(0);
        return 0;
}

int libworker_handle_service_reply(struct comm_point* ATTR_UNUSED(c), 
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(reply_info))
{
        log_assert(0);
        return 0;
}

void libworker_handle_control_cmd(struct tube* ATTR_UNUSED(tube),
        uint8_t* ATTR_UNUSED(buffer), size_t ATTR_UNUSED(len),
        int ATTR_UNUSED(error), void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

void libworker_fg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), 
        struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
        char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
{
        log_assert(0);
}

void libworker_bg_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), 
        struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
        char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
{
        log_assert(0);
}

void libworker_event_done_cb(void* ATTR_UNUSED(arg), int ATTR_UNUSED(rcode), 
        struct sldns_buffer* ATTR_UNUSED(buf), enum sec_status ATTR_UNUSED(s),
        char* ATTR_UNUSED(why_bogus), int ATTR_UNUSED(was_ratelimited))
{
        log_assert(0);
}

int context_query_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
{
        log_assert(0);
        return 0;
}

void worker_stat_timer_cb(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

void worker_probe_timer_cb(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

void worker_start_accept(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

void worker_stop_accept(void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

/** keep track of lock id in lock-verify application */
struct order_id {
        /** the thread id that created it */
        int thr;
        /** the instance number of creation */
        int instance;
};

int order_lock_cmp(const void* e1, const void* e2)
{
        const struct order_id* o1 = e1;
        const struct order_id* o2 = e2;
        if(o1->thr < o2->thr) return -1;
        if(o1->thr > o2->thr) return 1;
        if(o1->instance < o2->instance) return -1;
        if(o1->instance > o2->instance) return 1;
        return 0;
}

int
codeline_cmp(const void* a, const void* b)
{
        return strcmp(a, b);
}

int replay_var_compare(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
{
        log_assert(0);
        return 0;
}

void remote_get_opt_ssl(char* ATTR_UNUSED(str), void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

#ifdef USE_DNSTAP
void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
        void* ATTR_UNUSED(arg))
{
        log_assert(0);
}
#endif

#ifdef USE_DNSTAP
void dtio_mainfdcallback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
        void* ATTR_UNUSED(arg))
{
        log_assert(0);
}
#endif

void fast_reload_service_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
        void* ATTR_UNUSED(arg))
{
        log_assert(0);
}

int fast_reload_client_callback(struct comm_point* ATTR_UNUSED(c),
        void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
        struct comm_reply* ATTR_UNUSED(repinfo))
{
        log_assert(0);
        return 0;
}