Revision 1847 trunk/tests/test_download_queue.c

test_download_queue.c (revision 1847)
21 21
#include 
22 22

                
23 23
#include "common.h"
24
#include "test_download_queue.h"
25

                
24 26
#include "nntpgrab_utils.h"
25 27
#include "download_queue.h"
26 28

                
27
static Configuration *config = NULL;
28

                
29 29
/* Fake placeholders */
30 30
void
31 31
ng_plugin_emit_log_msg(NGPlugin *plugin_data, NGLogLevel log_level, const char *format, ...)
... ...
133 133
    ng_list_free(groups);
134 134
}
135 135

                
136
void
137
fill_queue_with_dummy_data(void)
138
{
139
    int i;
140
    char *errmsg = NULL;
141
    char *collection_name = "some collection";
142

                
143
    for (i = 1; i <= 10; i++) {
144
        char *subject;
145
        char *poster;
146
        time_t stamp;
147
        guint64 file_size = 0;
148
        NGList *list_parts;
149
        NGList *list_groups;
150

                
151
        subject = g_strdup_printf("file %i", i);
152
        poster = "some poster";
153
        stamp = 1185657196;
154

                
155
        list_parts = generate_part_list(&file_size);
156
        list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));
157
        list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));
158

                
159
        if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {
160
            FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);
161
            g_free(errmsg);
162
            return;
163
        }
164

                
165
        free_part_list(list_parts);
166
        free_group_list(list_groups);
167

                
168
        g_free(subject);
169
    }
170
}
171

                
172
void
173
drop_dummy_data_from_queue(void)
174
{
175
    int i;
176
    char *errmsg = NULL;
177
    char *collection_name = "some collection";
178

                
179
    for (i = 1; i <= 10; i++) {
180
        char *subject;
181

                
182
        subject = g_strdup_printf("file %i", i);
183

                
184
        if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {
185
            FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);
186
            g_free(errmsg);
187
            return;
188
        }
189

                
190
        g_free(subject);
191
    }
192
}
193

                
136 194
/* Testcases */
137 195
static void
138 196
test_download_queue_add_file_to_queue(void)
... ...
276 334
}
277 335

                
278 336
static void
279
test_download_queue_get_next_part_to_download_setup(void *data, gconstpointer user_data)
337
test_download_queue_setup_no_servers(void *data, gconstpointer user_data)
280 338
{
281
    int i;
282
    char *errmsg = NULL;
283
    char *collection_name = "some collection";
284

                
285
    for (i = 1; i <= 10; i++) {
286
        char *subject;
287
        char *poster;
288
        time_t stamp;
289
        guint64 file_size = 0;
290
        NGList *list_parts;
291
        NGList *list_groups;
292

                
293
        subject = g_strdup_printf("file %i", i);
294
        poster = "some poster";
295
        stamp = 1185657196;
296

                
297
        list_parts = generate_part_list(&file_size);
298
        list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));
299
        list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));
300

                
301
        if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {
302
            FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);
303
            g_free(errmsg);
304
            return;
305
        }
306

                
307
        free_part_list(list_parts);
308
        free_group_list(list_groups);
309

                
310
        g_free(subject);
311
    }
312
}
313

                
314
static void
315
test_download_queue_get_next_part_to_download_setup_no_servers(void *data, gconstpointer user_data)
316
{
317 339
    g_assert(configuration_get_avail_servers(config) == NULL);
318 340

                
319
    test_download_queue_get_next_part_to_download_setup(data, user_data);
341
    fill_queue_with_dummy_data();
320 342
}
321 343

                
322 344
static void
323
test_download_queue_get_next_part_to_download_setup_one_server_before(void *data, gconstpointer user_data)
345
test_download_queue_teardown_no_servers(void *data, gconstpointer user_data)
324 346
{
325
    char *errmsg = NULL;
326
    NGConfigServer server;
327

                
328 347
    g_assert(configuration_get_avail_servers(config) == NULL);
329 348

                
330
    memset(&server, 0, sizeof(server));
331
    strcpy(server.servername, "server1");
332
    strcpy(server.hostname, "host1");
333
    server.port = 119;
334
    server.enabled = TRUE;
335
    server.max_threads = 3;
336

                
337
    if (!configuration_add_server(config, server, &errmsg)) {
338
        FAIL_TEST("Unable to add server: %s\n", errmsg);
339
        g_free(errmsg);
340
        return;
341
    }
342

                
343
    test_download_queue_get_next_part_to_download_setup(data, user_data);
349
    drop_dummy_data_from_queue();
344 350
}
345 351

                
346 352
static void
347
test_download_queue_get_next_part_to_download_setup_one_server_after(void *data, gconstpointer user_data)
353
test_download_queue_no_servers(void *data, gconstpointer user_data)
348 354
{
349
    char *errmsg = NULL;
350
    NGConfigServer server;
351

                
352
    g_assert(configuration_get_avail_servers(config) == NULL);
353

                
354
    memset(&server, 0, sizeof(server));
355
    strcpy(server.servername, "server1");
356
    strcpy(server.hostname, "host1");
357
    server.port = 119;
358
    server.enabled = TRUE;
359
    server.max_threads = 1;
360

                
361
    test_download_queue_get_next_part_to_download_setup(data, user_data);
362

                
363
    if (!configuration_add_server(config, server, &errmsg)) {
364
        FAIL_TEST("Unable to add server: %s\n", errmsg);
365
        g_free(errmsg);
366
        return;
367
    }
368
}
369

                
370
static void
371
test_download_queue_get_next_part_to_download_teardown(void *data, gconstpointer user_data)
372
{
373
    int i;
374
    char *errmsg = NULL;
375
    char *collection_name = "some collection";
376

                
377
    for (i = 1; i <= 10; i++) {
378
        char *subject;
379

                
380
        subject = g_strdup_printf("file %i", i);
381

                
382
        if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {
383
            FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);
384
            g_free(errmsg);
385
            return;
386
        }
387

                
388
        g_free(subject);
389
    }
390
}
391

                
392
static void
393
test_download_queue_get_next_part_to_download_teardown_no_servers(void *data, gconstpointer user_data)
394
{
395
    g_assert(configuration_get_avail_servers(config) == NULL);
396

                
397
    test_download_queue_get_next_part_to_download_teardown(data, user_data);
398
}
399

                
400
static void
401
test_download_queue_get_next_part_to_download_teardown_one_server(void *data, gconstpointer user_data)
402
{
403
    char *errmsg = NULL;
404

                
405
    g_assert(configuration_get_avail_servers(config) != NULL);
406

                
407
    test_download_queue_get_next_part_to_download_teardown(data, user_data);
408

                
409
    if (!configuration_del_server(config, "server1", &errmsg)) {
410
        FAIL_TEST("Unable to remove server: %s\n", errmsg);
411
        g_free(errmsg);
412
        return;
413
    }
414
}
415

                
416
static void
417
test_download_queue_get_next_part_to_download_no_servers(void *data, gconstpointer user_data)
418
{
419 355
    NNTPCollection *collection = NULL;
420 356
    NNTPFile *file = NULL;
421 357
    NNTPPart *part = NULL;
... ...
433 369
    }
434 370
}
435 371

                
436
static void
437
test_download_queue_get_next_part_to_download_one_server_generic(gboolean all_available)
438
{
439
    /* There should be 10 files with 4 parts each in the download queue now */
440
    int i, j;
441

                
442
    NNTPCollection *prev_collection = NULL;
443
    NNTPFile *prev_file = NULL;
444

                
445
    NNTPCollection *collection = NULL;
446
    NNTPFile *file = NULL;
447
    NNTPPart *part = NULL;
448
    gboolean nothing_to_download_or_active = FALSE;
449

                
450
    GList *list, *files = NULL;
451

                
452
    for (i = 1; i <= 10; i++) {
453
        prev_file = NULL;
454

                
455
        for (j = 1; j <= 4; j++) {
456
            /* There 1 server configured at the moment, so a request for the next part for server_id 0
457
             * should return TRUE and nothing_do_download should_or_active be FALSE */
458
            if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
459
                g_test_message("nothing_to_download = %i", nothing_to_download_or_active);
460
                FAIL_TEST("download_queue_get_next_part_to_download returned FALSE while it isn't expected");
461
                return;
462
            } else {
463
                char *expected_subject = g_strdup_printf("file %i", i);
464

                
465
                g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);
466
                g_assert(collection != NULL);
467
                g_assert(file != NULL);
468
                g_assert(part != NULL);
469

                
470
                g_assert_cmpint(part->part_num, ==, j);
471
                g_assert_cmpstr(file->subject, ==, expected_subject);
472

                
473
                g_free(expected_subject);
474

                
475
                if (prev_collection == NULL) {
476
                    prev_collection = collection;
477
                } else {
478
                    g_assert(prev_collection == collection);
479
                }
480

                
481
                if (prev_file == NULL) {
482
                    prev_file = file;
483
                    files = g_list_append(files, file);
484
                } else {
485
                    g_assert(prev_file == file);
486
                }
487
            }
488

                
489
            /* Update the status of the part */
490
            if (!all_available && j % 2 == 0) {
491
                /* Indicate the every second part isn't available on the server and it shouldn't be retried again */
492
                download_queue_update_part_status("server1", 0, collection, file, part, 0, FALSE, FALSE, TRUE);
493
            } else {
494
                /* Indicate that the file was downloaded succesfully */
495
                download_queue_update_part_status("server1", 0, collection, file, part, 0, TRUE, FALSE, FALSE);
496
            }
497

                
498
            collection = NULL;
499
            file = NULL;
500
            part = NULL;
501
        }
502
    }
503

                
504
    /* We processed all files now. If we try to request another part it should indicate that there's nothing to download anymore */
505
    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
506
        /* Nothing is returned which is good */
507
        /* The flag nothing_to_download_or_active should be FALSE as all files are downloaded, but not decoded yet */
508
        g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);
509
        g_assert(collection == NULL);
510
        g_assert(file == NULL);
511
        g_assert(part == NULL);
512
    } else {
513
        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");
514
        return;
515
    }
516

                
517
    /* Mark all files as finished (downloaded+decoded) */
518
    list = files;
519
    while (list) {
520
        file = (NNTPFile*) list->data;
521

                
522
        if (!all_available && g_list_index(files, file) % 2 == 0) {
523
            /* Indicate that not all parts were available */
524
            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_INCOMPLETE);
525
        } else {
526
            /* Indicate that all parts were decoded succesfully */
527
            download_queue_mark_file_as_decoded_and_unref(prev_collection, file, "somefilename", TASK_STATE_FINISHED_COMPLETE);
528
        }
529

                
530
        list = g_list_next(list);
531
    }
532
    g_list_free(files);
533

                
534
    /* Verify that nothing_to_download_or_active is TRUE now */
535
    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
536
        /* Nothing is returned which is good */
537
        g_assert_cmpint(nothing_to_download_or_active, ==, TRUE);
538
        g_assert(collection == NULL);
539
        g_assert(file == NULL);
540
        g_assert(part == NULL);
541
    } else {
542
        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");
543
        return;
544
    }
545
}
546

                
547
static void
548
test_download_queue_get_next_part_to_download_one_server_all_available(void *data, gconstpointer user_data)
549
{
550
    /* Assume for this testcase that all parts are available on the server */
551
    test_download_queue_get_next_part_to_download_one_server_generic(TRUE);
552
}
553

                
554
static void
555
test_download_queue_get_next_part_to_download_one_server_some_available(void *data, gconstpointer user_data)
556
{
557
    /* Assume for this testcase that some parts are available on the server */
558
    test_download_queue_get_next_part_to_download_one_server_generic(FALSE);
559
}
560

                
561 372
int
562 373
main(int argc, char **argv)
563 374
{
... ...
583 394
    g_test_add_func("/download_queue/add_file_to_queue", test_download_queue_add_file_to_queue);
584 395
    g_test_add_func("/download_queue/restart_file", test_download_queue_restart_file);
585 396
    g_test_add_func("/download_queue/del_file_from_queue", test_download_queue_del_file_from_queue);
586
    g_test_add("/download_queue/get_next_part_to_download/no_servers", void, NULL,
587
               test_download_queue_get_next_part_to_download_setup_no_servers,
588
               test_download_queue_get_next_part_to_download_no_servers,
589
               test_download_queue_get_next_part_to_download_teardown_no_servers);
590
    g_test_add("/download_queue/get_next_part_to_download/one_server/before/all_available", void, NULL,
591
               test_download_queue_get_next_part_to_download_setup_one_server_before,
592
               test_download_queue_get_next_part_to_download_one_server_all_available,
593
               test_download_queue_get_next_part_to_download_teardown_one_server);
594
    g_test_add("/download_queue/get_next_part_to_download/one_server/after/all_available", void, NULL,
595
               test_download_queue_get_next_part_to_download_setup_one_server_after,
596
               test_download_queue_get_next_part_to_download_one_server_all_available,
597
               test_download_queue_get_next_part_to_download_teardown_one_server);
598
    g_test_add("/download_queue/get_next_part_to_download/one_server/before/some_available", void, NULL,
599
               test_download_queue_get_next_part_to_download_setup_one_server_before,
600
               test_download_queue_get_next_part_to_download_one_server_some_available,
601
               test_download_queue_get_next_part_to_download_teardown_one_server);
602
    g_test_add("/download_queue/get_next_part_to_download/one_server/after/some_available", void, NULL,
603
               test_download_queue_get_next_part_to_download_setup_one_server_after,
604
               test_download_queue_get_next_part_to_download_one_server_all_available,
605
               test_download_queue_get_next_part_to_download_teardown_one_server);
606 397

                
398
    /* No servers */
399
    g_test_add("/download_queue/no_servers", void, NULL,
400
               test_download_queue_setup_no_servers,
401
               test_download_queue_no_servers,
402
               test_download_queue_teardown_no_servers);
403

                
404
    /* One server */
405
    g_test_add("/download_queue/one_server/before/all_available", void, NULL,
406
               test_download_queue_setup_one_server_before,
407
               test_download_queue_one_server_all_available,
408
               test_download_queue_teardown_one_server);
409
    g_test_add("/download_queue/one_server/after/all_available", void, NULL,
410
               test_download_queue_setup_one_server_after,
411
               test_download_queue_one_server_all_available,
412
               test_download_queue_teardown_one_server);
413
    g_test_add("/download_queue/one_server/before/some_available", void, NULL,
414
               test_download_queue_setup_one_server_before,
415
               test_download_queue_one_server_some_available,
416
               test_download_queue_teardown_one_server);
417
    g_test_add("/download_queue/one_server/after/some_available", void, NULL,
418
               test_download_queue_setup_one_server_after,
419
               test_download_queue_one_server_all_available,
420
               test_download_queue_teardown_one_server);
421

                
422
    /* Two servers with equal priority */
423
    g_test_add("/download_queue/two_servers_equal_prio/before/all_available_on_one", void, NULL,
424
               test_download_queue_setup_two_servers_equal_prio_before,
425
               test_download_queue_two_servers_equal_prio_all_available_on_one,
426
               test_download_queue_teardown_two_servers);
427
    g_test_add("/download_queue/two_servers_equal_prio/after/all_available_on_one", void, NULL,
428
               test_download_queue_setup_two_servers_equal_prio_after,
429
               test_download_queue_two_servers_equal_prio_all_available_on_one,
430
               test_download_queue_teardown_two_servers);
431

                
432
    g_test_add("/download_queue/two_servers_equal_prio/before/all_available_on_one/server_two_added_between", void, NULL,
433
               test_download_queue_setup_two_servers_before_one_initial,
434
               test_download_queue_two_servers_equal_prio_all_available_on_one_server_two_added_between,
435
               test_download_queue_teardown_two_servers);
436
    g_test_add("/download_queue/two_servers_equal_prio/after/all_available_on_one/server_two_added_between", void, NULL,
437
               test_download_queue_setup_two_servers_after_one_initial,
438
               test_download_queue_two_servers_equal_prio_all_available_on_one_server_two_added_between,
439
               test_download_queue_teardown_two_servers);
440

                
441
    g_test_add("/download_queue/two_servers_equal_prio/before/some_available_on_one_all_on_two", void, NULL,
442
               test_download_queue_setup_two_servers_equal_prio_before,
443
               test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two,
444
               test_download_queue_teardown_two_servers);
445
    g_test_add("/download_queue/two_servers_equal_prio/after/some_available_on_one_all_on_two", void, NULL,
446
               test_download_queue_setup_two_servers_equal_prio_after,
447
               test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two,
448
               test_download_queue_teardown_two_servers);
449

                
450
    g_test_add("/download_queue/two_servers_equal_prio/before/some_available_on_one_all_on_two/server_two_added_between", void, NULL,
451
               test_download_queue_setup_two_servers_before_one_initial,
452
               test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two_server_two_added_between,
453
               test_download_queue_teardown_two_servers);
454
    g_test_add("/download_queue/two_servers_equal_prio/after/some_available_on_one_all_on_two/server_two_added_between", void, NULL,
455
               test_download_queue_setup_two_servers_after_one_initial,
456
               test_download_queue_two_servers_equal_prio_some_available_on_one_all_on_two_server_two_added_between,
457
               test_download_queue_teardown_two_servers);
458

                
459
    g_test_add("/download_queue/two_servers_equal_prio/before/some_available_on_one_some_on_two", void, NULL,
460
               test_download_queue_setup_two_servers_equal_prio_before,
461
               test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two,
462
               test_download_queue_teardown_two_servers);
463
    g_test_add("/download_queue/two_servers_equal_prio/after/some_available_on_one_some_on_two", void, NULL,
464
               test_download_queue_setup_two_servers_equal_prio_after,
465
               test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two,
466
               test_download_queue_teardown_two_servers);
467

                
468
    g_test_add("/download_queue/two_servers_equal_prio/before/some_available_on_one_some_on_two/server_two_added_between", void, NULL,
469
               test_download_queue_setup_two_servers_before_one_initial,
470
               test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two_server_two_added_between,
471
               test_download_queue_teardown_two_servers);
472
    g_test_add("/download_queue/two_servers_equal_prio/after/some_available_on_one_some_on_two/server_two_added_between", void, NULL,
473
               test_download_queue_setup_two_servers_after_one_initial,
474
               test_download_queue_two_servers_equal_prio_some_available_on_one_some_on_two_server_two_added_between,
475
               test_download_queue_teardown_two_servers);
476

                
477
    /* Two servers with different priority */
478
    g_test_add("/download_queue/two_servers_different_prio/before/all_available_on_one", void, NULL,
479
               test_download_queue_setup_two_servers_different_prio_before,
480
               test_download_queue_two_servers_different_prio_all_available_on_one,
481
               test_download_queue_teardown_two_servers);
482
    g_test_add("/download_queue/two_servers_different_prio/after/all_available_on_one", void, NULL,
483
               test_download_queue_setup_two_servers_different_prio_after,
484
               test_download_queue_two_servers_different_prio_all_available_on_one,
485
               test_download_queue_teardown_two_servers);
486

                
487
    g_test_add("/download_queue/two_servers_different_prio/before/all_available_on_one/server_two_added_between", void, NULL,
488
               test_download_queue_setup_two_servers_before_one_initial,
489
               test_download_queue_two_servers_different_prio_all_available_on_one_server_two_added_between,
490
               test_download_queue_teardown_two_servers);
491
    g_test_add("/download_queue/two_servers_different_prio/after/all_available_on_one/server_two_added_between", void, NULL,
492
               test_download_queue_setup_two_servers_after_one_initial,
493
               test_download_queue_two_servers_different_prio_all_available_on_one_server_two_added_between,
494
               test_download_queue_teardown_two_servers);
495

                
496
    g_test_add("/download_queue/two_servers_different_prio/before/some_available_on_one_all_on_two", void, NULL,
497
               test_download_queue_setup_two_servers_different_prio_before,
498
               test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two,
499
               test_download_queue_teardown_two_servers);
500
    g_test_add("/download_queue/two_servers_different_prio/after/some_available_on_one_all_on_two", void, NULL,
501
               test_download_queue_setup_two_servers_different_prio_after,
502
               test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two,
503
               test_download_queue_teardown_two_servers);
504

                
505
    g_test_add("/download_queue/two_servers_different_prio/before/some_available_on_one_all_on_two/server_two_added_between", void, NULL,
506
               test_download_queue_setup_two_servers_before_one_initial,
507
               test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two_server_two_added_between,
508
               test_download_queue_teardown_two_servers);
509
    g_test_add("/download_queue/two_servers_different_prio/after/some_available_on_one_all_on_two/server_two_added_between", void, NULL,
510
               test_download_queue_setup_two_servers_after_one_initial,
511
               test_download_queue_two_servers_different_prio_some_available_on_one_all_on_two_server_two_added_between,
512
               test_download_queue_teardown_two_servers);
513

                
514
    g_test_add("/download_queue/two_servers_different_prio/before/some_available_on_one_some_on_two", void, NULL,
515
               test_download_queue_setup_two_servers_different_prio_before,
516
               test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two,
517
               test_download_queue_teardown_two_servers);
518
    g_test_add("/download_queue/two_servers_different_prio/after/some_available_on_one_some_on_two", void, NULL,
519
               test_download_queue_setup_two_servers_different_prio_after,
520
               test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two,
521
               test_download_queue_teardown_two_servers);
522

                
523
    g_test_add("/download_queue/two_servers_different_prio/before/some_available_on_one_some_on_two/server_two_added_between", void, NULL,
524
               test_download_queue_setup_two_servers_before_one_initial,
525
               test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two_server_two_added_between,
526
               test_download_queue_teardown_two_servers);
527
    g_test_add("/download_queue/two_servers_different_prio/after/some_available_on_one_some_on_two/server_two_added_between", void, NULL,
528
               test_download_queue_setup_two_servers_after_one_initial,
529
               test_download_queue_two_servers_different_prio_some_available_on_one_some_on_two_server_two_added_between,
530
               test_download_queue_teardown_two_servers);
531

                
607 532
    /* Perform the tests */
608 533
    ret = g_test_run();
609 534

                

Also available in: Unified diff