NNTPGrab

/* 

    Copyright (C) 2005-2011  Erik van Pienbroek

    This program is free software; you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by

    the Free Software Foundation; either version 2 of the License, or

    (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License

    along with this program; if not, write to the Free Software

    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

#include 

#include 

#include 

#include "common.h"

#include "test_download_queue.h"

#include "nntpgrab_utils.h"

#include "download_queue.h"

static void

add_servers(gboolean add_server_before, gboolean equal_priority)

{

    char *errmsg = NULL;

    NGConfigServer server;

    g_assert(configuration_get_avail_servers(config) == NULL);

    if (!add_server_before) fill_queue_with_dummy_data();

    memset(&server, 0, sizeof(server));

    strcpy(server.servername, "server1");

    strcpy(server.hostname, "host1");

    server.port = 119;

    server.enabled = TRUE;

    server.max_threads = 3;

    server.priority = SERVER_PRIORITY_HIGH;

    if (!configuration_add_server(config, server, &errmsg)) {

        FAIL_TEST("Unable to add server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    memset(&server, 0, sizeof(server));

    strcpy(server.servername, "server2");

    strcpy(server.hostname, "host2");

    server.port = 119;

    server.enabled = TRUE;

    server.max_threads = 3;

    server.priority = (equal_priority ? SERVER_PRIORITY_HIGH : SERVER_PRIORITY_NORMAL);

    if (!configuration_add_server(config, server, &errmsg)) {

        FAIL_TEST("Unable to add server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    if (add_server_before) fill_queue_with_dummy_data();

}

void

test_download_queue_setup_two_servers_equal_prio_before(void *data, gconstpointer user_data)

{

    add_servers(TRUE, TRUE);

}

void

test_download_queue_setup_two_servers_different_prio_before(void *data, gconstpointer user_data)

{

    add_servers(TRUE, FALSE);

}

void

test_download_queue_setup_two_servers_equal_prio_after(void *data, gconstpointer user_data)

{

    add_servers(FALSE, TRUE);

}

void

test_download_queue_setup_two_servers_different_prio_after(void *data, gconstpointer user_data)

{

    add_servers(FALSE, FALSE);

}

void

test_download_queue_setup_two_servers_before_one_initial(void *data, gconstpointer user_data)

{

    test_download_queue_setup_one_server_before(data, user_data);

}

void

test_download_queue_setup_two_servers_after_one_initial(void *data, gconstpointer user_data)

{

    test_download_queue_setup_one_server_after(data, user_data);

}

void

test_download_queue_teardown_two_servers(void *data, gconstpointer user_data)

{

    char *errmsg = NULL;

    g_assert(configuration_get_avail_servers(config) != NULL);

    drop_dummy_data_from_queue();

    if (!configuration_del_server(config, "server2", &errmsg)) {

        FAIL_TEST("Unable to remove server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

    if (!configuration_del_server(config, "server1", &errmsg)) {

        FAIL_TEST("Unable to remove server: %s\n", errmsg);

        g_free(errmsg);

        return;

    }

}

static void

test_download_queue_two_servers_generic(gboolean all_available_on_one, gboolean all_available_on_two, gboolean equal_priority, gboolean server_added_between)

{

    /* There should be 10 files with 4 parts each in the download queue now */

    int i, server_id;

    int num_iterations, num_iterations_server1, num_servers;

    NNTPCollection *prev_collection = NULL;

    NNTPFile *prev_file = NULL;

    NNTPCollection *collection = NULL;

    NNTPFile *file = NULL;

    NNTPPart *part = NULL;

    gboolean nothing_to_download_or_active = FALSE;

    gboolean task_returned;

    gboolean server2_available = !server_added_between;

    GList *list, *files = NULL;

    if (equal_priority && all_available_on_one) {

        num_iterations = 40;

    } else if (equal_priority && !all_available_on_one) {

        num_iterations = 54;

    } else {

        num_iterations = 80;

    }

    num_iterations_server1 = 0;

    num_servers = (server_added_between ? 1 : 2);

    g_test_message("Num iterations for this testcase = %i", num_iterations);

    g_test_message("Server2 available = %i", server2_available);

    for (i = 0; i < num_iterations; i++) {

        g_test_message("Iteration %i", i);

        /* Try server 1 first, then server 2 (if it's available), then server 1, etc */

        if (num_iterations_server1 == 40) {     /* Don't try to consult server 1 anymore once it has been tried for all files+parts */

            server_id = 1;

        } else if (server2_available) {

            server_id = i % 2;

        } else {

            server_id = 0;

        }

        task_returned = download_queue_get_next_part_to_download(&collection, &file, &part, server_id, ¬hing_to_download_or_active);

        if (!task_returned) {

            if (server_id == 1) {

                /* This is okay */

                /* TODO: Implement */

                g_test_message("Nothing was returned for server_id %i", server_id);

                continue;

            } else {

                g_test_message("i = %i", i);

                g_test_message("server_id = %i", server_id);

                g_test_message("nothing_to_download = %i", nothing_to_download_or_active);

                FAIL_TEST("download_queue_get_next_part_to_download returned FALSE while it isn't expected");

                return;

            }

        } else {

            g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);

            g_assert(collection != NULL);

            g_assert(file != NULL);

            g_assert(part != NULL);

            if (equal_priority) {

                if (all_available_on_one) {

                    char *expected_subject = g_strdup_printf("file %i", (int) (i / 4) + 1);

                    g_assert_cmpint(part->part_num, ==, (i % 4) + 1);

                    g_assert_cmpstr(file->subject, ==, expected_subject);

                    g_free(expected_subject);

                } else {

                    /* The first server has marked each 2nd file as being unavailable so it will 

                     * be retried by the second server */

                    /* TODO: Implement */

                }

            } else {

                /* TODO: Implement */

            }

            if (prev_collection == NULL) {

                prev_collection = collection;

            } else {

                g_assert(prev_collection == collection);

            }

            if (prev_file != file) {

                prev_file = file;

                files = g_list_append(files, file);

            }

        }

        /* Update the status of the part */

        if ((server_id == 0 && !all_available_on_one && num_iterations_server1 % 2 == 0) ||

             (server_id == 1 && !all_available_on_two && i % 4 == 0)) {

            /* Indicate the every second part isn't available on the server and it shouldn't be retried again */

            g_test_message("Subject '%s' part %i was marked as being unavailable for server_id %i", file->subject, part->part_num, server_id);

            download_queue_update_part_status("dummy", 0, collection, file, part, server_id, FALSE, FALSE, TRUE);

        } else {

            /* Indicate that the file was downloaded succesfully */

            g_test_message("Subject '%s' part %i was marked as download successfully for server_id %i", file->subject, part->part_num, server_id);

            download_queue_update_part_status("dummy", 0, collection, file, part, server_id, TRUE, FALSE, FALSE);

        }

        /* If we're halfway, add a new server if necessary */

        if (server_added_between && i == 10) {

            char *errmsg = NULL;

            NGConfigServer server;

            memset(&server, 0, sizeof(server));

            strcpy(server.servername, "server2");

            strcpy(server.hostname, "host2");

            server.port = 119;

            server.enabled = TRUE;

            server.max_threads = 1;

            if (!configuration_add_server(config, server, &errmsg)) {

                FAIL_TEST("Unable to add server: %s\n", errmsg);

                g_free(errmsg);

                return;

            }

        }

        if (server_id == 0) {

            num_iterations_server1++;

        }

    }

    /* We processed all files now. If we try to request another part it should indicate that there's nothing to download anymore */

    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {

        /* Nothing is returned which is good */

        /* The flag nothing_to_download_or_active should be FALSE as all files are downloaded, but not decoded yet */

        g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);

        g_assert(collection == NULL);

        g_assert(file == NULL);

        g_assert(part == NULL);

    } else {

        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");

        return;

    }

    /* Mark all files as finished (downloaded+decoded) */

    list = files;

    while (list) {

        file = (NNTPFile*) list->data;

        if (!all_available_on_one && g_list_index(files, file) % 2 == 0) {

            /* Indicate that not all parts were available */

            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_INCOMPLETE);

        } else {

            /* Indicate that all parts were decoded succesfully */

            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_COMPLETE);

        }

        list = g_list_next(list);

    }

    g_list_free(files);

    /* Verify that nothing_to_download_or_active is TRUE now */

    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {

        /* Nothing is returned which is good */

        g_assert_cmpint(nothing_to_download_or_active, ==, TRUE);

        g_assert(collection == NULL);

        g_assert(file == NULL);

        g_assert(part == NULL);

    } else {

        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");

        return;

    }

}

void

test_download_queue_two_servers_equal_prio_all_available_on_one(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that all parts are available on server one. 

     * Both servers have an equal priority */

    test_download_queue_two_servers_generic(TRUE, FALSE, TRUE, FALSE);

}

void

test_download_queue_two_servers_equal_prio_all_available_on_one_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that all parts are available on server one 

     * and that a new server is added halfway. Both servers have an equal priority */

    test_download_queue_two_servers_generic(TRUE, FALSE, TRUE, TRUE);

}

void

test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and the rest is all available on server two. Both servers have an equal priority */

    test_download_queue_two_servers_generic(FALSE, TRUE, TRUE, FALSE);

}

void

test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and the rest is available on server two which is added halfway.

     * Both servers have an equal priority */

    test_download_queue_two_servers_generic(FALSE, TRUE, TRUE, TRUE);

}

void

test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and some are available on server two. Both servers have an equal priority */

    test_download_queue_two_servers_generic(FALSE, FALSE, TRUE, FALSE);

}

void

test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and some are available on server two which is added halfway.

     * Both servers have an equal priority */

    test_download_queue_two_servers_generic(FALSE, FALSE, TRUE, TRUE);

}

void

test_download_queue_two_servers_different_prio_all_available_on_one(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that all parts are available on server one. 

     * Both servers have a different priority */

    test_download_queue_two_servers_generic(TRUE, FALSE, FALSE, FALSE);

}

void

test_download_queue_two_servers_different_prio_all_available_on_one_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that all parts are available on server one 

     * and that a new server is added halfway. Both servers have a different priority */

    test_download_queue_two_servers_generic(TRUE, FALSE, FALSE, TRUE);

}

void

test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and the rest is all available on server two. Both servers have a different priority */

    test_download_queue_two_servers_generic(FALSE, TRUE, FALSE, FALSE);

}

void

test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and the rest is available on server two which is added halfway.

     * Both servers have a different priority */

    test_download_queue_two_servers_generic(FALSE, TRUE, FALSE, TRUE);

}

void

test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and some are available on server two. Both servers have a different priority */

    test_download_queue_two_servers_generic(FALSE, FALSE, FALSE, FALSE);

}

void

test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two_server_two_added_between(void *data, gconstpointer user_data)

{

    /* Assume for this testcase that some parts are available on server one 

     * and some are available on server two which is added halfway.

     * Both servers have a different priority */

    test_download_queue_two_servers_generic(FALSE, FALSE, FALSE, TRUE);

}