Revision 1843

trunk/nntpgrab_core/download_queue.c (revision 1843)
229 229
    }
230 230
}
231 231

                
232
static void
233
start_to_decode_if_necessary(NNTPCollection *collection, NNTPFile *file)
234
{
235
    /* Check if all parts belonging to this part are downloaded so we can start decoding it */
236
    GList *list = file->parts;
237
    gboolean complete_flag = TRUE;
238
    while (list) {
239
        NNTPPart *part = list->data;
240

                
241
        if (part->now_downloading) {
242
            file->file_is_downloading = TRUE;
243
            complete_flag = FALSE;
244
        }
245

                
246
        if (!part->downloaded) {
247
            complete_flag = FALSE;
248
        }
249

                
250
        list = g_list_next(list);
251
    }
252

                
253
    /* All the parts are already downloaded? */
254
    if (complete_flag && !file->now_decoding && !file->file_is_downloaded) {
255
        DecoderData *decoder_data;
256

                
257
        ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_DEBUG, "File '%s' from collection '%s' is now ready to be decoded", file->subject, collection->collection_name);
258

                
259
        file->now_decoding = TRUE;
260

                
261
        nntpgrab_core_emit_file_state_changed(FALSE, collection->collection_name, file->subject, NULL, file->status, TASK_STATE_WAITING_FOR_DECODE);
262

                
263
        file->status = TASK_STATE_WAITING_FOR_DECODE;
264

                
265
        decoder_data = g_slice_new0(DecoderData);
266
        decoder_data->collection = collection;
267
        decoder_data->file = file;
268
        //decoder_data->poolPar2verify = poolPar2verify;
269
        //decoder_data->poolPar2repair = poolPar2repair;
270
        //decoder_data->poolUnpack = poolUnpack;
271

                
272
        file_ref(file);
273
        collection_ref(collection);
274

                
275
        decoder_thread_push_task(decoder_data);
276
    }
277
}
278

                
279
static void
280
check_if_files_are_ready_to_be_decoded(void)
281
{
282
    GList *list;
283

                
284
    g_static_mutex_lock(&mutex);
285

                
286
    list = download_queue;
287
    while (list) {
288
        NNTPCollection *collection = list->data;
289
        GList *list2;
290

                
291
        list2 = collection->files;
292
        while (list2) {
293
            NNTPFile *file = list2->data;
294

                
295
            start_to_decode_if_necessary(collection, file);
296

                
297
            list2 = g_list_next(list2);
298
        }
299

                
300
        list = g_list_next(list);
301
    }
302
    g_static_mutex_unlock(&mutex);
303
}
304

                
232 305
static gboolean
233 306
on_config_changed(Configuration *config, gpointer data)
234 307
{
... ...
298 371

                
299 372
    if (!queue_rawfile_load(&download_queue, errmsg)) {
300 373
        /* Ignore read errors for now */
301
        ;
374
        g_free(*errmsg);
375
        *errmsg = NULL;
302 376
    }
303 377

                
304 378
    g_signal_connect(config, "config_changed", G_CALLBACK(on_config_changed), NULL);
... ...
362 436
}
363 437

                
364 438
gboolean
365
download_queue_get_next_part_to_download(NNTPCollection **collection, NNTPFile **file, NNTPPart **part, int server_id, gboolean *nothing_to_download)
439
download_queue_get_next_part_to_download(NNTPCollection **collection, NNTPFile **file, NNTPPart **part, int server_id, gboolean *nothing_to_download_or_active)
366 440
{
367 441
    GList *list;
442
    static gboolean beenhere = FALSE;
368 443

                
369 444
    g_return_val_if_fail(collection != NULL, FALSE);
370 445
    g_return_val_if_fail(file != NULL, FALSE);
371 446
    g_return_val_if_fail(part != NULL, FALSE);
372 447
    g_return_val_if_fail(server_id >= 0, FALSE);
373
    g_return_val_if_fail(nothing_to_download != NULL, FALSE);
448
    g_return_val_if_fail(nothing_to_download_or_active != NULL, FALSE);
374 449

                
450
    /* If we're here for the first time, check if there any decode-tasks ready */
451
    if (!beenhere) {
452
        beenhere = TRUE;
453
        check_if_files_are_ready_to_be_decoded();
454
    }
455

                
375 456
    ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_INFO, "download_queue_get_next_part_to_download for server_id %i", server_id);
376 457

                
377
    if (!server_positions[server_id].task_probably_ready) {
378
        /* It's uncertain whether there's anything left do download */
379
        *nothing_to_download = FALSE;
458
    *nothing_to_download_or_active = TRUE;
380 459

                
381
        return FALSE;
382
    }
383

                
384
    *nothing_to_download = TRUE;
385

                
386 460
    if (num_enabled_servers == 0) {
387 461
        /* No enabled servers found, don't even bother checking the queue */
388 462
        return FALSE;
... ...
390 464

                
391 465
    g_static_mutex_lock(&mutex);
392 466

                
467
    if (!server_positions[server_id].task_probably_ready) {
468
        /* It's uncertain whether there's anything left to download */
469
        /* Do only a quick scan of the now_decoding flag */
470
        goto check_decoded_state;
471
    }
472

                
393 473
    /* Do we already have a cached position for this server? */
394 474
    if (server_positions[server_id].collection_list) {
395 475
        list = server_positions[server_id].collection_list;
... ...
417 497

                
418 498
        while (list2) {
419 499
            GList *list3;
420
            gboolean complete_flag;
421 500

                
422 501
            *file = (NNTPFile *) list2->data;
423 502

                
... ...
447 526
                continue;
448 527
            }
449 528

                
450
            complete_flag = TRUE;
451

                
452 529
            /* Do we already have a cached position for this server? */
453 530
            if (server_positions[server_id].part_list) {
454 531
                list3 = server_positions[server_id].part_list;
... ...
460 537
                *part = (NNTPPart *) list3->data;
461 538

                
462 539
                if ((*part)->now_downloading) {
463
                    complete_flag = FALSE;
464
                    *nothing_to_download = FALSE;
540
                    *nothing_to_download_or_active = FALSE;
465 541
                }
466 542

                
467 543
                if (!(*part)->downloaded) {
... ...
475 551
                        server_positions[server_id].collection_pos = g_list_index(download_queue, *collection);
476 552
                        server_positions[server_id].file_pos = g_list_index((*collection)->files, *file);
477 553
                        server_positions[server_id].part_pos = g_list_index((*file)->parts, *part);
554
                        server_positions[server_id].task_probably_ready = TRUE;
478 555

                
479 556
                        g_static_mutex_unlock(&mutex);
480 557

                
481
                        *nothing_to_download = FALSE;
558
                        *nothing_to_download_or_active = FALSE;
482 559

                
483 560
                        file_ref(*file);
484 561
                        collection_ref(*collection);
... ...
488 565
                    }
489 566

                
490 567
                    // Are all the servers already tried ?
491
                    if ((is_server_enabled & (*part)->servers_already_tried) == is_server_enabled) {
492
                        complete_flag = TRUE;
493
                    } else {
494
                        complete_flag = FALSE;
495
                        *nothing_to_download = FALSE;
568
                    if (!(is_server_enabled & (*part)->servers_already_tried) == is_server_enabled) {
569
                        *nothing_to_download_or_active = FALSE;
496 570
                    }
497 571
                }
498 572

                
499 573
                list3 = g_list_next(list3);
500 574
            }
501 575

                
502
            // All the parts are already downloaded?
503
            if (complete_flag && !(*file)->now_decoding) {
504
                DecoderData *decoder_data;
576
            server_positions[server_id].part_list = NULL;
505 577

                
506
                ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_DEBUG, "File '%s' from collection '%s' is now ready to be decoded", (*file)->subject, (*collection)->collection_name);
578
            list2 = g_list_next(list2);
579
        }
507 580

                
508
                (*file)->now_decoding = TRUE;
581
        server_positions[server_id].file_list = NULL;
509 582

                
510
                nntpgrab_core_emit_file_state_changed(FALSE, (*collection)->collection_name, (*file)->subject, NULL, (*file)->status, TASK_STATE_WAITING_FOR_DECODE);
583
        list = g_list_next(list);
584
    }
511 585

                
512
                (*file)->status = TASK_STATE_WAITING_FOR_DECODE;
586
check_decoded_state:
513 587

                
514
                decoder_data = g_slice_new0(DecoderData);
515
                decoder_data->collection = (*collection);
516
                decoder_data->file = (*file);
517
                //decoder_data->poolPar2verify = poolPar2verify;
518
                //decoder_data->poolPar2repair = poolPar2repair;
519
                //decoder_data->poolUnpack = poolUnpack;
588
    /* Make sure that there's nothing in the 'waiting for decode' state */
589
    list = download_queue;
590
    while (list) {
591
        NNTPCollection *collection_tmp = (NNTPCollection *) list->data;
592
        GList *list2;
520 593

                
521
                file_ref(*file);
522
                collection_ref(*collection);
594
        // Is this collection being removed ?
595
        if (collection_tmp->stop_flag) {
596
            list = g_list_next(list);
597
            continue;
598
        }
523 599

                
524
                decoder_thread_push_task(decoder_data);
600
        list2 = collection_tmp->files_to_download;
601
        while (list2) {
602
            NNTPFile *file_tmp = (NNTPFile *) list2->data;
603

                
604
            // Is this file being removed?
605
            if (file_tmp->stop_flag) {
606
                list2 = g_list_next(list2);
607
                continue;
525 608
            }
526 609

                
610
            if (file_tmp->now_decoding) {
611
                *nothing_to_download_or_active = FALSE;
612
            }
613

                
527 614
            list2 = g_list_next(list2);
528 615
        }
529 616

                
530 617
        list = g_list_next(list);
531 618
    }
532 619

                
620
    if (*nothing_to_download_or_active == TRUE && server_positions[server_id].task_probably_ready == FALSE) {
621
        /* There was no task ready for this server, so we skipped the tests for looking up new tasks
622
         * and only checked if there any tasks being decoded at the moment. As there probably still
623
         * are downloads going on, we need to reset the nothing_to_download_or_active flag to FALSE
624
         * to avoid that the schedular gets paused */
625
        *nothing_to_download_or_active = FALSE;
626
    }
627

                
533 628
    *collection = NULL;
534 629
    *file = NULL;
535 630
    *part = NULL;
... ...
545 640
void
546 641
download_queue_update_part_status(const char *servername, int connection_id, NNTPCollection *collection, NNTPFile *file, NNTPPart *part, int server_id, gboolean success, gboolean reset_to_zero, gboolean dont_retry_anymore)
547 642
{
548
    GList *list;
549

                
550 643
    g_return_if_fail(servername != NULL);
551 644
    g_return_if_fail(connection_id >= 0);
552 645
    g_return_if_fail(collection != NULL);
... ...
560 653
    part->downloaded = success;
561 654
    file->file_is_downloading = FALSE;
562 655

                
563
    if (!file->file_is_new) {
564
        file->file_is_changed = TRUE;
565
    }
566

                
567 656
    if (success) {
568 657
        nntpgrab_core_emit_part_done(FALSE, servername, connection_id, collection->collection_name, file->subject, part->part_num, part->size);
569 658

                
... ...
601 690
        update_server_position_cache_if_necessary(collection, file, part, server_id);
602 691
    }
603 692

                
604
    // Are other threads downloading parts of this file ?
605
    list = file->parts;
606
    while (list) {
607
        NNTPPart *part2 = list->data;
608

                
609
        if (part2->now_downloading) {
610
            file->file_is_downloading = TRUE;
611
        }
612

                
613
        list = g_list_next(list);
614
    }
615

                
616
    // Are all the servers already tried ?
693
    /* Are all the servers already tried ? */
617 694
    if (!success && dont_retry_anymore) {
618 695
        gboolean all_servers_tried = TRUE;
619 696

                
... ...
623 700
            all_servers_tried = FALSE;
624 701
        }
625 702

                
626
        nntpgrab_core_emit_part_failed(FALSE, servername,connection_id, collection->collection_name, file->subject, part->part_num, part->size, all_servers_tried);
703
        nntpgrab_core_emit_part_failed(FALSE, servername, connection_id, collection->collection_name, file->subject, part->part_num, part->size, all_servers_tried);
627 704

                
628 705
        if (all_servers_tried && !reset_to_zero) {
629 706
            if (file->file_size_remaining < part->size) {
... ...
662 739
        }
663 740
    }
664 741

                
742
    start_to_decode_if_necessary(collection, file);
743

                
665 744
    g_static_mutex_unlock(&collection->mutex);
666 745
}
667 746

                
747
gboolean
748
download_queue_test_is_decoding_still_needed(NNTPCollection *collection, NNTPFile *file)
749
{
750
    g_return_val_if_fail(collection != NULL, FALSE);
751
    g_return_val_if_fail(file != NULL, FALSE);
752

                
753
    g_static_mutex_lock(&collection->mutex);
754
    if (file->stop_flag) {
755
        file->now_decoding = FALSE;
756
        g_static_mutex_unlock(&collection->mutex);
757

                
758
        file_unref(file);
759
        collection_unref(collection);
760

                
761
        return FALSE;
762
    }
763
    g_static_mutex_unlock(&collection->mutex);
764

                
765
    return TRUE;
766
}
767

                
768
void
769
download_queue_about_to_start_decoding(NNTPCollection *collection, NNTPFile *file)
770
{
771
    g_return_if_fail(collection != NULL);
772
    g_return_if_fail(file != NULL);
773

                
774
    nntpgrab_core_emit_file_state_changed(FALSE, collection->collection_name, file->subject, NULL, file->status, TASK_STATE_DECODING);
775

                
776
    g_static_mutex_lock(&collection->mutex);
777
    file->status = TASK_STATE_DECODING;
778
    g_static_mutex_unlock(&collection->mutex);
779
}
780

                
781
void
782
download_queue_mark_file_as_decoded_and_unref(NNTPCollection *collection, NNTPFile *file, const char *real_filename, NGTaskState status)
783
{
784
    g_return_if_fail(collection != NULL);
785
    g_return_if_fail(file != NULL);
786
    /* real_filename can be NULL */
787

                
788
    g_static_mutex_lock(&collection->mutex);
789

                
790
    /* Was the task restarted while the decoding was busy? */
791
    if (!file->now_decoding) {
792
        /* Yes it was, forget the status update */
793
        goto out;
794
    }
795

                
796
    if (status == TASK_STATE_FINISHED_COMPLETE || status == TASK_STATE_FINISHED_INCOMPLETE) {
797
        g_return_if_fail(real_filename != NULL);
798

                
799
        /* Update the real filename */
800
        strncpy((char*) file->real_filename, real_filename, sizeof(file->real_filename) - 1);
801

                
802
        if (file->file_type == NNTP_FILE_TYPE_UNKNOWN) {
803
            /* Check the extension again. It might have changed now that the decoding is finished */
804
            file->file_type = nntpgrab_utils_get_file_type_of_filename(file->real_filename);
805
        }
806

                
807
        file->file_is_downloaded = TRUE;
808
    } else if (status == TASK_STATE_WAITING_FOR_DECODE) {
809
        /* The decoder wants us to re-queue the file. Probably some temporary error occurs (like disk full) */
810

                
811
    }
812

                
813
    nntpgrab_core_emit_file_state_changed(FALSE, collection->collection_name, file->subject, file->real_filename, file->status, status);
814
    file->status = status;
815

                
816
    file->now_decoding = FALSE;
817

                
818
out:
819
    g_static_mutex_unlock(&collection->mutex);
820

                
821
    download_queue_test_is_collection_downloaded(collection);
822

                
823
    file_unref(file);
824
    collection_unref(collection);
825
}
826

                
668 827
static gboolean
669 828
test_is_subject_already_in_collection(NNTPCollection *collection, const char *subject)
670 829
{
... ...
741 900
            }
742 901

                
743 902
            file->position = g_list_index(collection->files, file) + 1;
744
            file->file_is_new = TRUE;
745 903

                
746 904
            nntpgrab_core_emit_file_added(FALSE, collection->collection_name, file->subject, file->poster, file->stamp, file->file_size, collection->total_size, collection->total_size_remaining, file->status, file->num_parts, file->groups);
747 905

                
... ...
751 909
                    // The poster wasn't set for this collection yet
752 910
                    strncpy(collection->poster, file->poster, sizeof(collection->poster) - 1);
753 911
                    nntpgrab_core_emit_collection_modified(FALSE, collection_name, collection->poster);
754
                    if (!collection->collection_is_new) {
755
                        collection->collection_is_changed = TRUE;
756
                    }
757 912
                }
758 913
            } else {
759 914
                // Some other file contained a different poster, reset to empty
760 915
                if (strcmp(collection->poster, file->poster)) {
761 916
                    memset(collection->poster, 0, sizeof(collection->poster));
762 917
                    nntpgrab_core_emit_collection_modified(FALSE, collection_name, collection->poster);
763
                    if (!collection->collection_is_new) {
764
                        collection->collection_is_changed = TRUE;
765
                    }
766 918
                }
767 919
            }
768 920

                
... ...
793 945
    } else {
794 946
        collection->total_size_remaining = 0;
795 947
    }
796
    collection->collection_is_new = TRUE;
797
    file->file_is_new = TRUE;
798 948
    file->position = 1;
799 949

                
800 950
    nntpgrab_core_emit_file_added(FALSE, collection->collection_name, file->subject, file->poster, file->stamp, file->file_size, collection->total_size, collection->total_size_remaining, file->status, file->num_parts, file->groups);
... ...
812 962
    gboolean ret;
813 963
    NNTPFile *file;
814 964
    GList *list;
965
#ifndef DOWNLOAD_QUEUE_DONT_SET_TMP_FILENAME
815 966
    char *tmp;
816 967
    char *tmp2;
968
#endif
817 969
    GHashTable *hashtable;
818 970
    char collection_name_copy[256];          /* This parameter needs to be of the same size as defined in base/collections.h! */
819 971

                
... ...
834 986
    file->file_size = file_size;
835 987
    file->file_size_remaining = file_size;
836 988

                
989
#ifndef DOWNLOAD_QUEUE_DONT_SET_TMP_FILENAME
837 990
    tmp = tempnam(opts.temp_directory, NULL);
838 991
    tmp2 = g_path_get_basename(tmp);
839 992
    strncpy((char *) file->tmp_filename, tmp2, sizeof(file->tmp_filename));
840 993
    g_free(tmp2);
841 994
    free(tmp);                  // Don't use g_free here as tempnam is a libc function!
995
#endif
842 996

                
843 997
    // Detect the file type and the par2 blocks
844 998
    if (!nntpgrab_utils_strip_subject(file->subject, NULL, NULL, NULL, NULL, NULL, &file->file_type, &file->par2_start_num, &file->par2_num_blocks, NULL, &file->num_parts)) {
... ...
1086 1240
                                // The poster wasn't set for this collection yet
1087 1241
                                strncpy(collection->poster, file->poster, sizeof(collection->poster) - 1);
1088 1242
                                nntpgrab_core_emit_collection_modified(FALSE, collection_name_copy, collection->poster);
1089

                
1090
                                if (!collection->collection_is_new) {
1091
                                    collection->collection_is_changed = TRUE;
1092
                                }
1093 1243
                            }
1094 1244
                        } else {
1095 1245
                            // Some other file contained a different poster, reset to empty
1096 1246
                            if (strcmp(collection->poster, file->poster)) {
1097 1247
                                memset(collection->poster, 0, sizeof(collection->poster));
1098 1248
                                nntpgrab_core_emit_collection_modified(FALSE, collection_name_copy, collection->poster);
1099

                
1100
                                if (!collection->collection_is_new) {
1101
                                    collection->collection_is_changed = TRUE;
1102
                                }
1103 1249
                            }
1104 1250
                        }
1105 1251

                
... ...
1167 1313
    int pos_last_file_to_download;
1168 1314
    NNTPCollection *collection;
1169 1315
    char collection_name_copy[256];          /* This parameter needs to be of the same size as defined in base/collections.h! */
1316
    int i;
1170 1317

                
1171 1318
    g_return_val_if_fail(collection_name != NULL, FALSE);
1172 1319
    /* subject can be NULL */
... ...
1249 1396
                    collection->total_size_remaining += file->file_size - file->file_size_remaining - filesize_remaining_active;
1250 1397
                    file->file_size_remaining = file->file_size - filesize_remaining_active;
1251 1398

                
1252
                    if (!file->file_is_new) {
1253
                        file->file_is_changed = TRUE;
1254
                    }
1255

                
1256 1399
                    nntpgrab_core_emit_file_state_changed(FALSE, collection_name_copy, file->subject, NULL, file->status, TASK_STATE_WAITING_FOR_DOWNLOAD);
1257 1400
                    file->status = TASK_STATE_WAITING_FOR_DOWNLOAD;
1258 1401

                
... ...
1261 1404

                
1262 1405
                        download_thread_awaken_if_needed();
1263 1406

                
1407
                        /* We can't rely on the contents of the server positions cache, so reset it */
1408
                        for (i = 0; i < MAX_NNTP_SERVERS; i++) {
1409
                            reset_server_positions(i);
1410
                        }
1411

                
1264 1412
                        return TRUE;
1265 1413
                    }
1266 1414
                }
... ...
1278 1426

                
1279 1427
                download_thread_awaken_if_needed();
1280 1428

                
1429
                /* We can't rely on the contents of the server positions cache, so reset it */
1430
                for (i = 0; i < MAX_NNTP_SERVERS; i++) {
1431
                    reset_server_positions(i);
1432
                }
1433

                
1281 1434
                return TRUE;
1282 1435
            }
1283 1436
        }
... ...
1462 1615

                
1463 1616
        if (file_tmp->position != i) {
1464 1617
            file_tmp->position = i;
1465
            file_tmp->position_is_changed = TRUE;
1466 1618
        }
1467 1619

                
1468 1620
        i++;
... ...
1478 1630

                
1479 1631
            if (file_tmp->position != i) {
1480 1632
                file_tmp->position = i;
1481
                file_tmp->position_is_changed = TRUE;
1482 1633
            }
1483 1634

                
1484 1635
            i++;
... ...
1541 1692
            download_queue = g_list_remove(download_queue, collection_src);
1542 1693
            download_queue = g_list_insert(download_queue, collection_src, new_position);
1543 1694

                
1544
            collection_src->collection_is_changed = TRUE;
1545

                
1546
            if (orig_position < new_position) {
1547
                /* Update the collections FROM the original position on */
1548
                NNTPCollection *collection = g_list_nth_data(download_queue, orig_position);
1549
                collection->collection_is_changed = TRUE;
1550
            }
1551

                
1552 1695
            break;
1553 1696
        }
1554 1697

                
trunk/nntpgrab_core/download_queue.h (revision 1843)
26 26
gboolean     download_queue_destroy(char **errmsg);
27 27
void         download_queue_reset_all_server_positions(void);
28 28
gboolean     download_queue_save(char **errmsg);
29
gboolean     download_queue_get_next_part_to_download(NNTPCollection **collection, NNTPFile **file, NNTPPart **part, int server_id, gboolean *nothing_to_download);
29
gboolean     download_queue_get_next_part_to_download(NNTPCollection **collection, NNTPFile **file, NNTPPart **part, int server_id, gboolean *nothing_to_download_or_active);
30 30
void         download_queue_update_part_status(const char *servername, int connection_id, NNTPCollection *collection, NNTPFile *file, NNTPPart *part, int server_id, gboolean success, gboolean reset_to_zero, gboolean dont_retry_anymore);
31
gboolean     download_queue_test_is_decoding_still_needed(NNTPCollection *collection, NNTPFile *file);
32
void         download_queue_about_to_start_decoding(NNTPCollection *collection, NNTPFile *file);
33
void         download_queue_mark_file_as_decoded_and_unref(NNTPCollection *collection, NNTPFile *file, const char *real_filename, NGTaskState status);
31 34
gboolean     download_queue_add_file_to_queue(const char *collection_name, const char *subject, const char *poster, time_t stamp, nguint64 file_size, NGList *groups, NGList *parts, char **errmsg);
32 35
gboolean     download_queue_del_file_from_queue(const char *collection_name, const char *subject, char **errmsg);
33 36
gboolean     download_queue_restart_file(const char *collection_name, const char *subject, char **errmsg);
trunk/nntpgrab_core/nntpgrab_internal.h (revision 1843)
22 22
#include 
23 23

                
24 24
#include "nntpgrab_types.h"
25
#include "collections.h"
26 25

                
27 26
void     core_signals_clear_stack(void);
28 27

                
trunk/nntpgrab_core/decoder_thread.c (revision 1843)
30 30
#include "nntpgrab.h"
31 31
#include "nntpgrab_internal.h"
32 32
#include "decoder_thread.h"
33
#include "collection_alloc.h"
34 33
#include "download_thread.h"
35 34
#include "download_queue.h"
36 35
#include "nntpgrab_plugin.h"
... ...
43 42
static NGConfigOpts opts;
44 43

                
45 44
static NNTPGrabDecoderRes
46
decoder_thread_decode_file(const char *collection_name, NNTPFile *file, int *saved_errno, char **errmsg)
45
decoder_thread_decode_file(const char *collection_name, NNTPFile *file, char **real_filename, int *saved_errno, char **errmsg)
47 46
{
48 47
    NNTPGrabDecoderRes retval;
49 48

                
... ...
54 53

                
55 54
    ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_INFO, _("Trying to decode file with from collection '%s' with subject = '%s'"), collection_name, file->subject);
56 55

                
57
    g_return_val_if_fail(ng_plugin_call(NULL, "decode_file", collection_name, file, opts.temp_directory, opts.download_directory, saved_errno, errmsg, &retval), FALSE);
56
    g_return_val_if_fail(ng_plugin_call(NULL, "decode_file", collection_name, file, opts.temp_directory, opts.download_directory, real_filename, saved_errno, errmsg, &retval), FALSE);
58 57

                
59 58
    ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_INFO, _("Decoder returned %i"), retval);
60 59

                
... ...
62 61
        ng_plugin_emit_log_msg(NULL, NG_LOG_LEVEL_INFO, "Decoder error: %s", *errmsg);
63 62
    }
64 63

                
65
    // The filename got updated, check the extension again
66
    if (retval != DECODER_RESULT_ERROR &&
67
        retval != DECODER_RESULT_NO_PARTS_AVAIL &&
68
        file->file_type == NNTP_FILE_TYPE_UNKNOWN) {
69

                
70
        file->file_type = nntpgrab_utils_get_file_type_of_filename(file->real_filename);
71
    }
72

                
73 64
    return retval;
74 65
}
75 66

                
... ...
80 71
    int saved_errno = 0;
81 72
    static time_t last_timestamp = 0;
82 73
    NNTPGrabDecoderRes res;
74
    char *real_filename = NULL;
83 75
    char *errmsg = NULL;
76
    NGTaskState state;
84 77

                
85
    // Do we still need to decode?
86
    g_static_mutex_lock(&decoder_data->collection->mutex);
87
    if (abort_flag || decoder_data->file->stop_flag) {
88
        decoder_data->file->now_decoding = FALSE;
89
        g_static_mutex_unlock(&decoder_data->collection->mutex);
90

                
91
        file_unref(decoder_data->file);
92
        collection_unref(decoder_data->collection);
93

                
78
    /* Do we still need to decode? */
79
    if (abort_flag || !download_queue_test_is_decoding_still_needed(decoder_data->collection, decoder_data->file)) {
94 80
        g_slice_free(DecoderData, decoder_data);
95

                
96 81
        return;
97 82
    }
98
    g_static_mutex_unlock(&decoder_data->collection->mutex);
99 83

                
100
    nntpgrab_core_emit_file_state_changed(FALSE, decoder_data->collection->collection_name, decoder_data->file->subject, NULL, decoder_data->file->status, TASK_STATE_DECODING);
84
    download_queue_about_to_start_decoding(decoder_data->collection, decoder_data->file);
101 85

                
102
    g_static_mutex_lock(&decoder_data->collection->mutex);
103
    decoder_data->file->status = TASK_STATE_DECODING;
104
    g_static_mutex_unlock(&decoder_data->collection->mutex);
105

                
106 86
    // Perform the actual decoding
107
    if ((res = decoder_thread_decode_file(decoder_data->collection->collection_name, decoder_data->file, &saved_errno, &errmsg)) == DECODER_RESULT_ERROR) {
87
    if ((res = decoder_thread_decode_file(decoder_data->collection->collection_name, decoder_data->file, &real_filename, &saved_errno, &errmsg)) == DECODER_RESULT_ERROR) {
108 88
        if (saved_errno == ENOSPC) {
109 89
            // No free space available, stop the schedular
110 90
            if (errmsg) {
... ...
117 97
            /* Skip any already queued decodes because of the error */
118 98
            abort_flag = TRUE;
119 99

                
120
            g_static_mutex_lock(&decoder_data->collection->mutex);
121
            decoder_data->file->now_decoding = FALSE;
122
            nntpgrab_core_emit_file_state_changed(FALSE, decoder_data->collection->collection_name, decoder_data->file->subject, NULL, decoder_data->file->status, TASK_STATE_WAITING_FOR_DECODE);
123
            decoder_data->file->status = TASK_STATE_WAITING_FOR_DECODE;
124
            g_static_mutex_unlock(&decoder_data->collection->mutex);
125

                
126
            file_unref(decoder_data->file);
127
            collection_unref(decoder_data->collection);
128

                
100
            download_queue_mark_file_as_decoded_and_unref(decoder_data->collection, decoder_data->file, NULL, TASK_STATE_WAITING_FOR_DECODE);
129 101
            g_slice_free(DecoderData, decoder_data);
130 102

                
131 103
            return;
... ...
144 116
            /* Skip any already queued decodes because of the error */
145 117
            abort_flag = TRUE;
146 118

                
147
            g_static_mutex_lock(&decoder_data->collection->mutex);
148
            decoder_data->file->now_decoding = FALSE;
149
            nntpgrab_core_emit_file_state_changed(FALSE, decoder_data->collection->collection_name, decoder_data->file->subject, NULL, decoder_data->file->status, TASK_STATE_WAITING_FOR_DECODE);
150
            decoder_data->file->status = TASK_STATE_WAITING_FOR_DECODE;
151
            g_static_mutex_unlock(&decoder_data->collection->mutex);
152

                
153
            file_unref(decoder_data->file);
154
            collection_unref(decoder_data->collection);
155

                
119
            download_queue_mark_file_as_decoded_and_unref(decoder_data->collection, decoder_data->file, NULL, TASK_STATE_WAITING_FOR_DECODE);
156 120
            g_slice_free(DecoderData, decoder_data);
157 121

                
158 122
            return;
159 123
        }
160 124
    }
161 125

                
162
    // Mark the file as completed
163
    g_static_mutex_lock(&decoder_data->collection->mutex);
126
    /* Mark the file as completed */
127
    switch(res) {
128
        case DECODER_RESULT_INCOMPLETE:
129
            state = TASK_STATE_FINISHED_INCOMPLETE;
130
            break;
164 131

                
165
    if (!decoder_data->file->file_is_new) {
166
        decoder_data->file->file_is_changed = TRUE;
167
    }
132
        case DECODER_RESULT_NO_PARTS_AVAIL:
133
            state = TASK_STATE_FINISHED_NO_PARTS_AVAIL;
134
            break;
168 135

                
169
    // Was the task restarted while the decoding was busy?
170
    if (decoder_data->file->now_decoding) {
171
        // Nope, mark the file decode as completed
172
        NGTaskState state;
136
        case DECODER_RESULT_ERROR:
137
            state = TASK_STATE_WAITING_FOR_DECODE;
138
            break;
173 139

                
174
        decoder_data->file->file_is_downloaded = TRUE;
175
        decoder_data->file->now_decoding = FALSE;
176

                
177
        switch(res) {
178
            case DECODER_RESULT_INCOMPLETE:
179
                state = TASK_STATE_FINISHED_INCOMPLETE;
180
                break;
181

                
182
            case DECODER_RESULT_NO_PARTS_AVAIL:
183
                state = TASK_STATE_FINISHED_NO_PARTS_AVAIL;
184
                break;
185

                
186
            case DECODER_RESULT_ERROR:
187
                state = TASK_STATE_WAITING_FOR_DECODE;
188
                break;
189

                
190
            default:
191
            case DECODER_RESULT_COMPLETE:
192
                state = TASK_STATE_FINISHED_COMPLETE;
193
                break;
194
        }
195

                
196
        nntpgrab_core_emit_file_state_changed(FALSE, decoder_data->collection->collection_name, decoder_data->file->subject, decoder_data->file->real_filename, decoder_data->file->status, state);
197
        decoder_data->file->status = state;
140
        default:
141
        case DECODER_RESULT_COMPLETE:
142
            state = TASK_STATE_FINISHED_COMPLETE;
143
            break;
198 144
    }
199

                
200
    g_static_mutex_unlock(&decoder_data->collection->mutex);
201

                
202
    download_queue_test_is_collection_downloaded(decoder_data->collection);
203

                
204
    file_unref(decoder_data->file);
205
    collection_unref(decoder_data->collection);
206

                
145
    download_queue_mark_file_as_decoded_and_unref(decoder_data->collection, decoder_data->file, real_filename, state);
207 146
    g_slice_free(DecoderData, decoder_data);
147
    if (real_filename) {
148
        g_free(real_filename);
149
    }
208 150

                
209 151
    // Save the download queue only when there has been some time between now and the previous time we've decoded something
210 152
    if (time(NULL) > last_timestamp + 60) {
trunk/nntpgrab_core/queue_rawfile.c (revision 1843)
282 282
                    break;
283 283

                
284 284
                case TASK_STATE_WAITING_FOR_DOWNLOAD:
285
                    active_file->file_is_downloaded = FALSE;
286
                    break;
287

                
285 288
                case TASK_STATE_FINISHED_COMPLETE:
286 289
                case TASK_STATE_FINISHED_INCOMPLETE:
287 290
                case TASK_STATE_FINISHED_NO_PARTS_AVAIL:
trunk/tests/test_download_queue.c (revision 1843)
1
/*
2
    Copyright (C) 2005-2011  Erik van Pienbroek
3

                
4
    This program is free software; you can redistribute it and/or modify
5
    it under the terms of the GNU General Public License as published by
6
    the Free Software Foundation; either version 2 of the License, or
7
    (at your option) any later version.
8

                
9
    This program is distributed in the hope that it will be useful,
10
    but WITHOUT ANY WARRANTY; without even the implied warranty of
11
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
    GNU General Public License for more details.
13

                
14
    You should have received a copy of the GNU General Public License
15
    along with this program; if not, write to the Free Software
16
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
17
*/
18

                
19
#include 
20
#include 
21
#include 
22

                
23
#include "common.h"
24
#include "nntpgrab_utils.h"
25
#include "download_queue.h"
26

                
27
static Configuration *config = NULL;
28

                
29
/* Fake placeholders */
30
void
31
ng_plugin_emit_log_msg(NGPlugin *plugin_data, NGLogLevel log_level, const char *format, ...)
32
{
33
    va_list args;
34
    char msg[1024];
35
    va_start(args, format);
36
    memset(msg, 0, sizeof(msg));
37
    vsnprintf(msg, sizeof(msg) - 1, format, args);
38
    va_end(args);
39

                
40
    g_test_message("%s", msg);
41
}
42

                
43
typedef void* DecoderData;
44
void
45
decoder_thread_push_task(DecoderData *task)
46
{
47

                
48
}
49

                
50
NntpgrabCore *
51
get_core(void)
52
{
53
    return NULL;
54
}
55

                
56
void
57
download_thread_awaken_if_needed(void)
58
{
59

                
60
}
61

                
62
ngboolean
63
ng_plugin_emit_internal_event(const char *event_name, ...)
64
{
65
    return TRUE;
66
}
67

                
68
/* Helper functions */
69
static NGList *
70
generate_part_list(guint64 *file_size)
71
{
72
    NNTPPart *part;
73
    NGList *list;
74

                
75
    g_return_val_if_fail(file_size != NULL, NULL);
76

                
77
    *file_size = 0;
78

                
79
    part = g_slice_new0(NNTPPart);
80
    strncpy((char *) part->message_id, "", sizeof(part->message_id));
81
    part->size = 258374;
82
    part->part_num = 1;
83
    *file_size += part->size;
84
    list = ng_list_append(NULL, part);
85

                
86
    part = g_slice_new0(NNTPPart);
87
    strncpy((char *) part->message_id, "", sizeof(part->message_id));
88
    part->size = 258436;
89
    part->part_num = 2;
90
    *file_size += part->size;
91
    list = ng_list_append(list, part);
92

                
93
    part = g_slice_new0(NNTPPart);
94
    strncpy((char *) part->message_id, "", sizeof(part->message_id));
95
    part->size = 258498;
96
    part->part_num = 3;
97
    *file_size += part->size;
98
    list = ng_list_append(list, part);
99

                
100
    part = g_slice_new0(NNTPPart);
101
    strncpy((char *) part->message_id, "", sizeof(part->message_id));
102
    part->size = 258448;
103
    part->part_num = 4;
104
    *file_size += part->size;
105
    list = ng_list_append(list, part);
106

                
107
    return list;
108
}
109

                
110
void
111
free_part_list(NGList *parts)
112
{
113
    NGList *list = parts;
114

                
115
    while (list) {
116
        g_slice_free(NNTPPart, list->data);
117
        list = ng_list_next(list);
118
    }
119

                
120
    ng_list_free(parts);
121
}
122

                
123
void
124
free_group_list(NGList *groups)
125
{
126
    NGList *list = groups;
127

                
128
    while (list) {
129
        g_free(list->data);
130
        list = ng_list_next(list);
131
    }
132

                
133
    ng_list_free(groups);
134
}
135

                
136
/* Testcases */
137
static void
138
test_download_queue_add_file_to_queue(void)
139
{
140
    char *errmsg = NULL;
141
    NGList *list_parts;
142
    NGList *list_groups;
143
    char *collection_name;
144
    char *subject;
145
    char *poster;
146
    time_t stamp;
147
    guint64 file_size = 0;
148

                
149
    GError *err = NULL;
150
    char *filename_expected;
151
    char *filename_real;
152
    char *contents_expected;
153
    char *contents_real;
154

                
155
    /* Add a single file to the download queue */
156
    collection_name = "some collection";
157
    subject = "some subject";
158
    poster = "some poster";
159
    stamp = 1185657196;
160

                
161
    list_parts = generate_part_list(&file_size);
162
    list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));
163
    list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));
164

                
165
    if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {
166
        FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);
167
    }
168

                
169
    free_part_list(list_parts);
170
    free_group_list(list_groups);
171

                
172
    if (!download_queue_save(&errmsg)) {
173
        FAIL_TEST("Unable to save download queue: %s\n", errmsg);
174
        g_free(errmsg);
175
        return;
176
    }
177

                
178
    /* Verify the contents of the download queue file */
179
    filename_expected = "download_queue_with_1_file.db";
180
    if (!g_file_get_contents(filename_expected, &contents_expected, NULL, &err)) {
181
        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_expected, err->message);
182
        g_error_free(err);
183
        return;
184
    }
185

                
186
    filename_real = get_location_of_download_queue();
187
    if (!g_file_get_contents(filename_real, &contents_real, NULL, &err)) {
188
        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_real, err->message);
189
        g_free(contents_expected);
190
        g_free(filename_real);
191
        g_error_free(err);
192
        return;
193
    }
194
    g_free(filename_real);
195

                
196
    g_assert_cmpstr(contents_expected, ==, contents_real);
197

                
198
    g_free(contents_expected);
199
    g_free(contents_real);
200
}
201

                
202
static void
203
test_download_queue_restart_file(void)
204
{
205
    char *collection_name;
206
    char *subject;
207
    char *errmsg = NULL;
208

                
209
    g_test_message("Now trying to restart file which was added in the add_item_to_queue testcase");
210
    collection_name = "some collection";
211
    subject = "some subject";
212
    if (!download_queue_restart_file(collection_name, subject, &errmsg)) {
213
        FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);
214
    }
215

                
216
    g_test_message("Now trying to restart a non-existant file");
217
    if (!download_queue_restart_file("non-existant", subject, &errmsg)) {
218
        /* This is good */
219
        g_free(errmsg);
220
    } else {
221
        FAIL_TEST("download_queue_restart_file returned TRUE, while FALSE was expected");
222
        return;
223
    }
224

                
225
    /* Verify the contents of the download queue file */
226

                
227
}
228

                
229
static void
230
test_download_queue_del_file_from_queue(void)
231
{
232
    char *collection_name;
233
    char *subject;
234
    char *errmsg = NULL;
235
    GError *err = NULL;
236
    char *filename_real;
237
    char *contents_real = NULL;
238

                
239
    g_test_message("Now trying to remove file which was added in the add_item_to_queue testcase");
240
    collection_name = "some collection";
241
    subject = "some subject";
242
    if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {
243
        FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);
244
        g_free(errmsg);
245
        return;
246
    }
247

                
248
    g_test_message("Now trying to remove a non-existant file");
249
    if (!download_queue_del_file_from_queue("non-existant", subject, &errmsg)) {
250
        /* This is good */
251
        g_free(errmsg);
252
    } else {
253
        FAIL_TEST("download_queue_del_file_from_queue returned TRUE, while FALSE was expected");
254
        return;
255
    }
256

                
257
    if (!download_queue_save(&errmsg)) {
258
        FAIL_TEST("Unable to save download queue: %s\n", errmsg);
259
        g_free(errmsg);
260
        return;
261
    }
262

                
263
    /* Verify the contents of the download queue file */
264
    filename_real = get_location_of_download_queue();
265
    if (!g_file_get_contents(filename_real, &contents_real, NULL, &err)) {
266
        FAIL_TEST("Unable to read contents of '%s': %s\n", filename_real, err->message);
267
        g_free(filename_real);
268
        g_error_free(err);
269
        return;
270
    }
271
    g_free(filename_real);
272

                
273
    g_assert_cmpstr("", ==, contents_real);
274

                
275
    g_free(contents_real);
276
}
277

                
278
static void
279
test_download_queue_get_next_part_to_download_setup(void *data, gconstpointer user_data)
280
{
281
    int i;
282
    char *errmsg = NULL;
283
    char *collection_name = "some collection";
284

                
285
    for (i = 1; i <= 10; i++) {
286
        char *subject;
287
        char *poster;
288
        time_t stamp;
289
        guint64 file_size = 0;
290
        NGList *list_parts;
291
        NGList *list_groups;
292

                
293
        subject = g_strdup_printf("file %i", i);
294
        poster = "some poster";
295
        stamp = 1185657196;
296

                
297
        list_parts = generate_part_list(&file_size);
298
        list_groups = ng_list_append(NULL, g_strdup("alt.binaries.x"));
299
        list_groups = ng_list_append(list_groups, g_strdup("alt.binaries.nl"));
300

                
301
        if (!download_queue_add_file_to_queue(collection_name, subject, poster, stamp, file_size, list_groups, list_parts, &errmsg)) {
302
            FAIL_TEST("Unable to add file to download queue: %s\n", errmsg);
303
            g_free(errmsg);
304
            return;
305
        }
306

                
307
        free_part_list(list_parts);
308
        free_group_list(list_groups);
309

                
310
        g_free(subject);
311
    }
312
}
313

                
314
static void
315
test_download_queue_get_next_part_to_download_setup_no_servers(void *data, gconstpointer user_data)
316
{
317
    g_assert(configuration_get_avail_servers(config) == NULL);
318

                
319
    test_download_queue_get_next_part_to_download_setup(data, user_data);
320
}
321

                
322
static void
323
test_download_queue_get_next_part_to_download_setup_one_server_before(void *data, gconstpointer user_data)
324
{
325
    char *errmsg = NULL;
326
    NGConfigServer server;
327

                
328
    g_assert(configuration_get_avail_servers(config) == NULL);
329

                
330
    memset(&server, 0, sizeof(server));
331
    strcpy(server.servername, "server1");
332
    strcpy(server.hostname, "host1");
333
    server.port = 119;
334
    server.enabled = TRUE;
335
    server.max_threads = 3;
336

                
337
    if (!configuration_add_server(config, server, &errmsg)) {
338
        FAIL_TEST("Unable to add server: %s\n", errmsg);
339
        g_free(errmsg);
340
        return;
341
    }
342

                
343
    test_download_queue_get_next_part_to_download_setup(data, user_data);
344
}
345

                
346
static void
347
test_download_queue_get_next_part_to_download_setup_one_server_after(void *data, gconstpointer user_data)
348
{
349
    char *errmsg = NULL;
350
    NGConfigServer server;
351

                
352
    g_assert(configuration_get_avail_servers(config) == NULL);
353

                
354
    memset(&server, 0, sizeof(server));
355
    strcpy(server.servername, "server1");
356
    strcpy(server.hostname, "host1");
357
    server.port = 119;
358
    server.enabled = TRUE;
359
    server.max_threads = 1;
360

                
361
    test_download_queue_get_next_part_to_download_setup(data, user_data);
362

                
363
    if (!configuration_add_server(config, server, &errmsg)) {
364
        FAIL_TEST("Unable to add server: %s\n", errmsg);
365
        g_free(errmsg);
366
        return;
367
    }
368
}
369

                
370
static void
371
test_download_queue_get_next_part_to_download_teardown(void *data, gconstpointer user_data)
372
{
373
    int i;
374
    char *errmsg = NULL;
375
    char *collection_name = "some collection";
376

                
377
    for (i = 1; i <= 10; i++) {
378
        char *subject;
379

                
380
        subject = g_strdup_printf("file %i", i);
381

                
382
        if (!download_queue_del_file_from_queue(collection_name, subject, &errmsg)) {
383
            FAIL_TEST("Unable to remove file from download queue: %s\n", errmsg);
384
            g_free(errmsg);
385
            return;
386
        }
387

                
388
        g_free(subject);
389
    }
390
}
391

                
392
static void
393
test_download_queue_get_next_part_to_download_teardown_no_servers(void *data, gconstpointer user_data)
394
{
395
    g_assert(configuration_get_avail_servers(config) == NULL);
396

                
397
    test_download_queue_get_next_part_to_download_teardown(data, user_data);
398
}
399

                
400
static void
401
test_download_queue_get_next_part_to_download_teardown_one_server(void *data, gconstpointer user_data)
402
{
403
    char *errmsg = NULL;
404

                
405
    g_assert(configuration_get_avail_servers(config) != NULL);
406

                
407
    test_download_queue_get_next_part_to_download_teardown(data, user_data);
408

                
409
    if (!configuration_del_server(config, "server1", &errmsg)) {
410
        FAIL_TEST("Unable to remove server: %s\n", errmsg);
411
        g_free(errmsg);
412
        return;
413
    }
414
}
415

                
416
static void
417
test_download_queue_get_next_part_to_download_no_servers(void *data, gconstpointer user_data)
418
{
419
    NNTPCollection *collection = NULL;
420
    NNTPFile *file = NULL;
421
    NNTPPart *part = NULL;
422
    gboolean nothing_to_download = FALSE;
423

                
424
    /* There are no servers configured at the moment, so a request for the next part for server_id 0
425
     * should return FALSE and nothing_do_download should be TRUE */
426
    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download)) {
427
        g_assert_cmpint(nothing_to_download, ==, 1);
428
        g_assert(collection == NULL);
429
        g_assert(file == NULL);
430
        g_assert(part == NULL);
431
    } else {
432
        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");
433
    }
434
}
435

                
436
static void
437
test_download_queue_get_next_part_to_download_one_server_all_available(void *data, gconstpointer user_data)
438
{
439
    /* There should be 10 files with 4 parts each in the download queue now
440
     * Assume for this testcase that all parts are available on the server */
441
    int i, j;
442

                
443
    NNTPCollection *prev_collection = NULL;
444
    NNTPFile *prev_file = NULL;
445

                
446
    NNTPCollection *collection = NULL;
447
    NNTPFile *file = NULL;
448
    NNTPPart *part = NULL;
449
    gboolean nothing_to_download_or_active = FALSE;
450

                
451
    GList *list, *files = NULL;
452

                
453
    for (i = 0; i < 10; i++) {
454
        prev_file = NULL;
455

                
456
        for (j = 0; j < 4; j++) {
457
            /* There 1 server configured at the moment, so a request for the next part for server_id 0
458
             * should return TRUE and nothing_do_download should_or_active be FALSE */
459
            if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
460
                g_test_message("nothing_to_download = %i", nothing_to_download_or_active);
461
                FAIL_TEST("download_queue_get_next_part_to_download returned FALSE while it isn't expected");
462
                return;
463
            } else {
464
                g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);
465
                g_assert(collection != NULL);
466
                g_assert(file != NULL);
467
                g_assert(part != NULL);
468

                
469
                if (prev_collection == NULL) {
470
                    prev_collection = collection;
471
                } else {
472
                    g_assert(prev_collection == collection);
473
                }
474

                
475
                if (prev_file == NULL) {
476
                    prev_file = file;
477
                    files = g_list_append(files, file);
478
                } else {
479
                    g_assert(prev_file == file);
480
                }
481
            }
482

                
483
            /* Update the status of the part so that everything got completed succesfully */
484
            download_queue_update_part_status("server1", 0, collection, file, part, 0, TRUE, FALSE, FALSE);
485

                
486
            collection = NULL;
487
            file = NULL;
488
            part = NULL;
489
        }
490
    }
491

                
492
    /* We processed all files now. If we try to request another part it should indicate that there's nothing to download anymore */
493
    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
494
        /* Nothing is returned which is good */
495
        /* The flag nothing_to_download_or_active should be FALSE as all files are downloaded, but not decoded yet */
496
        g_assert_cmpint(nothing_to_download_or_active, ==, FALSE);
497
        g_assert(collection == NULL);
498
        g_assert(file == NULL);
499
        g_assert(part == NULL);
500
    } else {
501
        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");
502
        return;
503
    }
504

                
505
    /* Mark all files as finished (downloaded+decoded) */
506
    list = files;
507
    while (list) {
508
        file = (NNTPFile*) list->data;
509

                
510
        file->file_is_downloaded = TRUE;
511
        file->now_decoding = FALSE;
512
        file->status = TASK_STATE_FINISHED_COMPLETE;
513

                
514
        list = g_list_next(list);
515
    }
516
    g_list_free(files);
517

                
518
    /* Verify that nothing_to_download_or_active is TRUE now */
519
    if (!download_queue_get_next_part_to_download(&collection, &file, &part, 0, ¬hing_to_download_or_active)) {
520
        /* Nothing is returned which is good */
521
        g_assert_cmpint(nothing_to_download_or_active, ==, TRUE);
522
        g_assert(collection == NULL);
523
        g_assert(file == NULL);
524
        g_assert(part == NULL);
525
    } else {
526
        FAIL_TEST("download_queue_get_next_part_to_download returned TRUE while it isn't expected");
527
        return;
528
    }
529
}
530

                
531
int
532
main(int argc, char **argv)
533
{
534
    int ret;
535
    char *errmsg = NULL;
536

                
537
    g_type_init();
538
    g_test_init(&argc, &argv, NULL);
539
    g_test_bug_base("https://www.nntpgrab.nl/issues/");
540

                
541
    initialize_tests();
542

                
543
    /* Base initialisation for download queue operations */
544
    if (!(config = configuration_new(&errmsg, NULL))) {
545
        g_error("Unable to initialize configuration. Unable to continue download queue tests: %s\n", errmsg);
546
    }
547

                
548
    if (!download_queue_init(config, &errmsg)) {
549
        g_error("Unable to initialize download queue. Unable to continue download queue tests: %s\n", errmsg);
550
    }
551

                
552
    /* Add the various testcases */
553
    g_test_add_func("/download_queue/add_file_to_queue", test_download_queue_add_file_to_queue);
554
    g_test_add_func("/download_queue/restart_file", test_download_queue_restart_file);
555
    g_test_add_func("/download_queue/del_file_from_queue", test_download_queue_del_file_from_queue);
556
    g_test_add("/download_queue/get_next_part_to_download/no_servers", void, NULL,
557
               test_download_queue_get_next_part_to_download_setup_no_servers,
558
               test_download_queue_get_next_part_to_download_no_servers,
559
               test_download_queue_get_next_part_to_download_teardown_no_servers);
560
    g_test_add("/download_queue/get_next_part_to_download/one_server/before/all_available", void, NULL,
561
               test_download_queue_get_next_part_to_download_setup_one_server_before,
562
               test_download_queue_get_next_part_to_download_one_server_all_available,
563
               test_download_queue_get_next_part_to_download_teardown_one_server);
564
    g_test_add("/download_queue/get_next_part_to_download/one_server/after/all_available", void, NULL,
565
               test_download_queue_get_next_part_to_download_setup_one_server_after,
566
               test_download_queue_get_next_part_to_download_one_server_all_available,
567
               test_download_queue_get_next_part_to_download_teardown_one_server);
568

                
569
    /* Perform the tests */
570
    ret = g_test_run();
571

                
572
    /* Cleanup */
573
    if (!download_queue_destroy(&errmsg)) {
574
        g_error("Unable to destroy download queue: %s\n", &errmsg);
575
    }
576

                
577
    configuration_destroy(config);
578

                
579
    cleanup_tests();
580

                
581
    return ret;
582
}
trunk/tests/valgrind.supp (revision 1843)
1
# Valgrind error suppression file
2

                
3
# ============================= libc ==================================
4

                
5
{
6
   ld.so initialization + selinux
7
   Memcheck:Leak
8
   ...
9
   fun:_dl_init
10
   obj:/lib/ld-*.so
11
}
12

                
13
{
14
   dlopen initialization, triggered by handle-leak-debug code
15
   Memcheck:Leak
16
   ...
17
   fun:__libc_dlopen_mode
18
   fun:init
19
   fun:backtrace
20
   fun:handle_leak_debug_bt
21
   fun:dynamic_ensure_handle
22
   fun:tp_handle_ensure
23
}
24

                
25
# default.supp has these for 2.10, but they're too specific
26
{
27
   Debian libc6 (2.10.x, 2.11.x) stripped dynamic linker
28
   Memcheck:Cond
29
   fun:index
30
   fun:expand_dynamic_string_token
31
   fun:_dl_map_object
32
   fun:map_doit
33
   fun:_dl_catch_error
34
   fun:do_preload
35
   fun:dl_main
36
   fun:_dl_sysdep_start
37
   fun:_dl_start
38
   obj:/lib/ld-*.so
39
}
40
{
41
   Debian libc6 (2.9.x - 2.11.x) stripped dynamic linker
42
   Memcheck:Cond
43
   fun:_dl_relocate_object
44
   fun:dl_main
45
   fun:_dl_sysdep_start
46
   fun:_dl_start
47
   obj:/lib/ld-*.so
48
}
49

                
50
{
51
   ld.so initialization on glibc 2.9
52
   Memcheck:Cond
53
   fun:strlen
54
   fun:_dl_init_paths
55
   fun:dl_main
56
   fun:_dl_sysdep_start
57
   fun:_dl_start
58
   obj:/lib/ld-2.9.so
59
}
60

                
61
# ======================= libselinux on Debian amd64 =====================
62

                
63
{
64
   I have no idea what SELinux is doing but it's not my problem
65
   Memcheck:Cond
66
   ...
67
   obj:/lib/libselinux.so.1
68
   obj:/lib/libselinux.so.1
69
   obj:/lib/libselinux.so.1
70
}
71

                
72
{
73
   I have no idea what SELinux is doing but it's not my problem
74
   Memcheck:Value8
75
   ...
76
   obj:/lib/libselinux.so.1
77
   obj:/lib/libselinux.so.1
78
   obj:/lib/libselinux.so.1
79
}
80

                
81
{
82
   I have no idea what SELinux is doing but it's not my problem
83
   Memcheck:Leak
84
   ...
85
   obj:/lib/libselinux.so.1
86
   obj:/lib/libselinux.so.1
87
   obj:/lib/libselinux.so.1
88
}
89

                
90
# ============================= GLib ==================================
91

                
92
{
93
   g_set_prgname copies its argument
94
   Memcheck:Leak
95
   ...
96
   fun:g_set_prgname
97
}
98

                
99
{
100
   one g_get_charset per child^Wprocess
101
   Memcheck:Leak
102
   ...
103
   fun:g_get_charset
104
}
105

                
106
{
107
   one g_get_home_dir per process
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff