NNTPGrab

/* 

    Copyright (C) 2005-2011  Erik van Pienbroek

    This program is free software; you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by

    the Free Software Foundation; either version 2 of the License, or

    (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License

    along with this program; if not, write to the Free Software

    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

#include 

#include 

#include 

#include "common.h"

#include "nntpgrab_utils.h"

#include "download_queue.h"

static Configuration *config = NULL;

/* Fake placeholders */

void

ng_plugin_emit_log_msg(NGPlugin *plugin_data, NGLogLevel log_level, const char *format, ...)

{

    va_list args;

    char msg[1024];

    va_start(args, format);

    memset(msg, 0, sizeof(msg));

    vsnprintf(msg, sizeof(msg) - 1, format, args);

    va_end(args);

    g_test_message("%s", msg);

}

typedef void* DecoderData;

void

decoder_thread_push_task(DecoderData *task)

{

}

NntpgrabCore *

get_core(void)

{

    return NULL;

}

void

download_thread_awaken_if_needed(void)

{

}

ngboolean

ng_plugin_emit_internal_event(const char *event_name, ...)

{

    return TRUE;

}

/* Helper functions */

static NGList *

generate_part_list(guint64 *file_size)

{

    NNTPPart *part;

    NGList *list;

    g_return_val_if_fail(file_size != NULL, NULL);

    *file_size = 0;

    part = g_slice_new0(NNTPPart);

    strncpy((char *) part->message_id, "", sizeof(part->message_id));

    part->size = 258374;

    part->part_num = 1;

    *file_size += part->size;

    list = ng_list_append(NULL, part);

    part = g_slice_new0(NNTPPart);

    strncpy((char *) part->message_id, "", sizeof(part->message_id));

    part->size = 258436;

    part->part_num = 2;

    *file_size += part->size;

    list = ng_list_append(list, part);

    part = g_slice_new0(NNTPPart);

    strncpy((char *) part->message_id, "", sizeof(part->message_id));

    part->size = 258498;

    part->part_num = 3;

    *file_size += part->size;

    list = ng_list_append(list, part);

    part = g_slice_new0(NNTPPart);

    strncpy((char *) part->message_id, "", sizeof(part->message_id));

    part->size = 258448;

    part->part_num = 4;

    *file_size += part->size;

    list = ng_list_append(list, part);

    return list;

}

void

free_part_list(NGList *parts)

{

    NGList *list = parts;

    while (list) {

        g_slice_free(NNTPPart, list->data);

        list = ng_list_next(list);

    }

    ng_list_free(parts);

}

void

free_group_list(NGList *groups)

{

    NGList *list = groups;

    while (list) {

        g_free(list->data);

        list = ng_list_next(list);

    }

    ng_list_free(groups);

}

/* Testcases */

static void

test_download_queue_add_file_to_queue(void)

{

    char *errmsg = NULL;

    NGList *list_parts;

    NGList *list_groups;

    char *collection_name;

    char *subject;

    char *poster;

    time_t stamp;

    guint64 file_size = 0;

    GError *err = NULL;

    char *filename_expected;

    char *filename_real;

    char *contents_expected;

    char *contents_real;

    /* Add a single file to the download queue */

    collection_name = "some collection";

    subject = "some subject";

    poster = "some poster";

    stamp = 1185657196;

    list_parts = generate_part_list(&file_size);

    list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));

    list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));

    if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {

        FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);

    }

    free_part_list(list_parts);

    free_group_list(list_groups);

    if (!download_queue_save(&errmsg)) {

        FAIL_TEST("Unable to save download queue: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    /* Verify the contents of the download queue file */

    filename_expected = "download_queue_with_1_file.db";

    if (!g_file_get_contents(filename_expected, &contents_expected, NULL, &err)) {

        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_expected, err->message);

        g_error_free(err);

        return;

    }

    filename_real = get_location_of_download_queue();

    if (!g_file_get_contents(filename_real, &contents_real, NULL, &err)) {

        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_real, err->message);

        g_free(contents_expected);

        g_free(filename_real);

        g_error_free(err);

        return;

    }

    g_free(filename_real);

    g_assert_cmpstr(contents_expected, ==, contents_real);

    g_free(contents_expected);

    g_free(contents_real);

}

static void

test_download_queue_restart_file(void)

{

    char *collection_name;

    char *subject;

    char *errmsg = NULL;

    g_test_message("Now trying to restart file which was added in the add_item_to_queue testcase");

    collection_name = "some collection";

    subject = "some subject";

    if (!download_queue_restart_file(collection_name, subject, &errmsg)) {

        FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);

    }

    g_test_message("Now trying to restart a non-existant file");

    if (!download_queue_restart_file("non-existant", subject, &errmsg)) {

        /* This is good */

        g_free(errmsg);

    } else {

        FAIL_TEST("download_queue_restart_file returned TRUE, while FALSE was expected");

        return;

    }

    /* Verify the contents of the download queue file */

}

static void

test_download_queue_del_file_from_queue(void)

{

    char *collection_name;

    char *subject;

    char *errmsg = NULL;

    GError *err = NULL;

    char *filename_real;

    char *contents_real = NULL;

    g_test_message("Now trying to remove file which was added in the add_item_to_queue testcase");

    collection_name = "some collection";

    subject = "some subject";

    if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {

        FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    g_test_message("Now trying to remove a non-existant file");

    if (!download_queue_del_file_from_queue("non-existant", subject, &errmsg)) {

        /* This is good */

        g_free(errmsg);

    } else {

        FAIL_TEST("download_queue_del_file_from_queue returned TRUE, while FALSE was expected");

        return;

    }

    if (!download_queue_save(&errmsg)) {

        FAIL_TEST("Unable to save download queue: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    /* Verify the contents of the download queue file */

    filename_real = get_location_of_download_queue();

    if (!g_file_get_contents(filename_real, &contents_real, NULL, &err)) {

        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_real, err->message);

        g_free(filename_real);

        g_error_free(err);

        return;

    }

    g_free(filename_real);

    g_assert_cmpstr("", ==, contents_real);

    g_free(contents_real);

}

static void

test_download_queue_get_next_part_to_download_setup(void *data, gconstpointer user_data)

{

    int i;

    char *errmsg = NULL;

    char *collection_name = "some collection";

    for (i = 1; i <= 10; i++) {

        char *subject;

        char *poster;

        time_t stamp;

        guint64 file_size = 0;

        NGList *list_parts;

        NGList *list_groups;

        subject = g_strdup_printf("file %i", i);

        poster = "some poster";

        stamp = 1185657196;

        list_parts = generate_part_list(&file_size);

        list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));

        list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));

        if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {

            FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);

            g_free(errmsg);

            return;

        }

        free_part_list(list_parts);

        free_group_list(list_groups);

        g_free(subject);

    }

}

static void

test_download_queue_get_next_part_to_download_setup_no_servers(void *data, gconstpointer user_data)

{

    g_assert(configuration_get_avail_servers(config) == NULL);

    test_download_queue_get_next_part_to_download_setup(data, user_data);

}

static void

test_download_queue_get_next_part_to_download_setup_one_server_before(void *data, gconstpointer user_data)

{

    char *errmsg = NULL;

    NGConfigServer server;

    g_assert(configuration_get_avail_servers(config) == NULL);

    memset(&server, 0, sizeof(server));

    strcpy(server.servername, "server1");

    strcpy(server.hostname, "host1");

    server.port = 119;

    server.enabled = TRUE;

    server.max_threads = 3;

    if (!configuration_add_server(config, server, &errmsg)) {

        FAIL_TEST("Unable to add server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    test_download_queue_get_next_part_to_download_setup(data, user_data);

}

static void

test_download_queue_get_next_part_to_download_setup_one_server_after(void *data, gconstpointer user_data)

{

    char *errmsg = NULL;

    NGConfigServer server;

    g_assert(configuration_get_avail_servers(config) == NULL);

    memset(&server, 0, sizeof(server));

    strcpy(server.servername, "server1");

    strcpy(server.hostname, "host1");

    server.port = 119;

    server.enabled = TRUE;

    server.max_threads = 1;

    test_download_queue_get_next_part_to_download_setup(data, user_data);

    if (!configuration_add_server(config, server, &errmsg)) {

        FAIL_TEST("Unable to add server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

}

static void

test_download_queue_get_next_part_to_download_teardown(void *data, gconstpointer user_data)

{

    int i;

    char *errmsg = NULL;

    char *collection_name = "some collection";

    for (i = 1; i <= 10; i++) {

        char *subject;

        subject = g_strdup_printf("file %i", i);

        if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {

            FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);

            g_free(errmsg);

            return;

        }

        g_free(subject);

    }

}

static void

test_download_queue_get_next_part_to_download_teardown_no_servers(void *data, gconstpointer user_data)

{

    g_assert(configuration_get_avail_servers(config) == NULL);

    test_download_queue_get_next_part_to_download_teardown(data, user_data);

}

static void

test_download_queue_get_next_part_to_download_teardown_one_server(void *data, gconstpointer user_data)

{

    char *errmsg = NULL;

    g_assert(configuration_get_avail_servers(config) != NULL);

    test_download_queue_get_next_part_to_download_teardown(data, user_data);

    if (!configuration_del_server(config, "server1", &errmsg)) {

        FAIL_TEST("Unable to remove server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

}

static void

test_download_queue_get_next_part_to_download_no_servers(void *data, gconstpointer user_data)

{

    NNTPCollection *collection = NULL;

    NNTPFile *file = NULL;

    NNTPPart *part = NULL;

    gboolean nothing_to_download = FALSE;

    /* There are no servers configured at the moment, so a request for the next part for server_id 0 

     * should return FALSE and nothing_do_download should be TRUE */

    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download)) {

        g_assert_cmpint(nothing_to_download, ==, 1);

        g_assert(collection == NULL);

        g_assert(file == NULL);

        g_assert(part == NULL);

    } else {

        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");

    }

}

static void

test_download_queue_get_next_part_to_download_one_server_generic(gboolean all_available)

{

    /* There should be 10 files with 4 parts each in the download queue now */

    int i, j;

    NNTPCollection *prev_collection = NULL;

    NNTPFile *prev_file = NULL;

    NNTPCollection *collection = NULL;

    NNTPFile *file = NULL;

    NNTPPart *part = NULL;

    gboolean nothing_to_download_or_active = FALSE;

    GList *list, *files = NULL;

    for (i = 1; i <= 10; i++) {

        prev_file = NULL;

        for (j = 1; j <= 4; j++) {

            /* There 1 server configured at the moment, so a request for the next part for server_id 0 

             * should return TRUE and nothing_do_download should_or_active be FALSE */

            if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {

                g_test_message("nothing_to_download = %i", nothing_to_download_or_active);

                FAIL_TEST("download_queue_get_next_part_to_download returned FALSE while it isn't expected");

                return;

            } else {

                char *expected_subject = g_strdup_printf("file %i", i);

                g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);

                g_assert(collection != NULL);

                g_assert(file != NULL);

                g_assert(part != NULL);

                g_assert_cmpint(part->part_num, ==, j);

                g_assert_cmpstr(file->subject, ==, expected_subject);

                g_free(expected_subject);

                if (prev_collection == NULL) {

                    prev_collection = collection;

                } else {

                    g_assert(prev_collection == collection);

                }

                if (prev_file == NULL) {

                    prev_file = file;

                    files = g_list_append(files, file);

                } else {

                    g_assert(prev_file == file);

                }

            }

            /* Update the status of the part */

            if (!all_available && j % 2 == 0) {

                /* Indicate the every second part isn't available on the server and it shouldn't be retried again */

                download_queue_update_part_status("server1", 0, collection, file, part, 0, FALSE, FALSE, TRUE);

            } else {

                /* Indicate that the file was downloaded succesfully */

                download_queue_update_part_status("server1", 0, collection, file, part, 0, TRUE, FALSE, FALSE);

            }

            collection = NULL;

            file = NULL;

            part = NULL;

        }

    }

    /* We processed all files now. If we try to request another part it should indicate that there's nothing to download anymore */

    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {

        /* Nothing is returned which is good */

        /* The flag nothing_to_download_or_active should be FALSE as all files are downloaded, but not decoded yet */

        g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);

        g_assert(collection == NULL);

        g_assert(file == NULL);

        g_assert(part == NULL);

    } else {

        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");

        return;

    }

    /* Mark all files as finished (downloaded+decoded) */

    list = files;

    while (list) {

        file = (NNTPFile*) list->data;

        if (!all_available && g_list_index(files, file) % 2 == 0) {

            /* Indicate that not all parts were available */

            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_INCOMPLETE);

        } else {

            /* Indicate that all parts were decoded succesfully */

            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_COMPLETE);

        }

        list = g_list_next(list);

    }

    g_list_free(files);

    /* Verify that nothing_to_download_or_active is TRUE now */

    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {

        /* Nothing is returned which is good */

        g_assert_cmpint(nothing_to_download_or_active, ==, TRUE);

        g_assert(collection == NULL);

        g_assert(file == NULL);

        g_assert(part == NULL);

    } else {

        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");

        return;

    }

}

static void

test_download_queue_get_next_part_to_download_one_server_all_available(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that all parts are available on the server */

    test_download_queue_get_next_part_to_download_one_server_generic(TRUE);

}

static void

test_download_queue_get_next_part_to_download_one_server_some_available(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on the server */

    test_download_queue_get_next_part_to_download_one_server_generic(FALSE);

}

int

main(int argc, char **argv)

{

    int ret;

    char *errmsg = NULL;

    g_type_init();

    g_test_init(&argc, &argv, NULL);

    g_test_bug_base("https://www.nntpgrab.nl/issues/");

    initialize_tests();

    /* Base initialisation for download queue operations */

    if (!(config = configuration_new(&errmsg, NULL))) {

        g_error("Unable to initialize configuration. Unable to continue download queue tests: %s\n", errmsg);

    }

    if (!download_queue_init(config, &errmsg)) {

        g_error("Unable to initialize download queue. Unable to continue download queue tests: %s\n", errmsg);

    }

    /* Add the various testcases */

    g_test_add_func("/download_queue/add_file_to_queue", test_download_queue_add_file_to_queue);

    g_test_add_func("/download_queue/restart_file", test_download_queue_restart_file);

    g_test_add_func("/download_queue/del_file_from_queue", test_download_queue_del_file_from_queue);

    g_test_add("/download_queue/get_next_part_to_download/no_servers", void, NULL,

               test_download_queue_get_next_part_to_download_setup_no_servers,

               test_download_queue_get_next_part_to_download_no_servers,

               test_download_queue_get_next_part_to_download_teardown_no_servers);

    g_test_add("/download_queue/get_next_part_to_download/one_server/before/all_available", void, NULL,

               test_download_queue_get_next_part_to_download_setup_one_server_before,

               test_download_queue_get_next_part_to_download_one_server_all_available,

               test_download_queue_get_next_part_to_download_teardown_one_server);

    g_test_add("/download_queue/get_next_part_to_download/one_server/after/all_available", void, NULL,

               test_download_queue_get_next_part_to_download_setup_one_server_after,

               test_download_queue_get_next_part_to_download_one_server_all_available,

               test_download_queue_get_next_part_to_download_teardown_one_server);

    g_test_add("/download_queue/get_next_part_to_download/one_server/before/some_available", void, NULL,

               test_download_queue_get_next_part_to_download_setup_one_server_before,

               test_download_queue_get_next_part_to_download_one_server_some_available,

               test_download_queue_get_next_part_to_download_teardown_one_server);

    g_test_add("/download_queue/get_next_part_to_download/one_server/after/some_available", void, NULL,

               test_download_queue_get_next_part_to_download_setup_one_server_after,

               test_download_queue_get_next_part_to_download_one_server_all_available,

               test_download_queue_get_next_part_to_download_teardown_one_server);

    /* Perform the tests */

    ret = g_test_run();

    /* Cleanup */

    if (!download_queue_destroy(&errmsg)) {

        g_error("Unable to destroy download queue: %s\n", &errmsg);

    }

    configuration_destroy(config);

    cleanup_tests();

    return ret;

}