queue.c

Go to the documentation of this file.

/*
 Copyright (C) 2005-2007 Erik van Pienbroek

 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

#ifdef _MSC_VER
#include "config.h.win32"
#else
#include "config.h"
#endif

#include 
#include 
#include 
#include 
#include 

#include "nntpgrab_plugin_schedular.h"
#include "nntpgrab_types.h"

#ifdef _MSC_VER
#undef atoll
#define atoll _atoi64
#endif

static char *xml_parser_errmsg = NULL;

static void
xml_parser_error_func (void *ctx, const char *msg, ...)
{
    va_list args;
    char *orig = xml_parser_errmsg;
    char *tmp;

    va_start(args, msg);

    tmp = g_strdup_vprintf(msg, args);
    if (orig) {
        xml_parser_errmsg = g_strdup_printf("%s %s", orig, tmp);
        g_free(orig);
        g_free(tmp);
    } else {
        xml_parser_errmsg = tmp;
    }

    va_end(args);
}

static char *
generate_filename(void)
{
    return g_build_filename(g_get_user_config_dir(), "NNTPGrab", "download_queue.xml", NULL);
}

static void
parse_part(xmlDocPtr doc, xmlNodePtr cur, NNTPFile *file, NNTPPart *part)
{
    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "MessageID")) {
            xmlChar *message_id = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (message_id) {
                strncpy((char *) part->message_id, (const char *) message_id, (size_t) sizeof(part->message_id));
                xmlFree(message_id);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Size")) {
            xmlChar *size = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (size) {
                part->size = atoi((const char *) size);
                xmlFree(size);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Downloaded")) {
            xmlChar *downloaded = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (downloaded) {
                part->downloaded = atoi((const char *) downloaded);
                xmlFree(downloaded);
            }
        }

        cur = cur->next;
    }
}

static void
parse_groups(xmlDocPtr doc, xmlNodePtr cur, NNTPFile *file)
{
    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "Group")) {
            xmlChar *group = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (group) {
                file->groups = g_list_append(file->groups, g_strdup((const gchar*) group));
                xmlFree(group);
            }
        }

        cur = cur->next;
    }
}

static void
parse_parts(xmlDocPtr doc, xmlNodePtr cur, NNTPFile *file)
{
    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "Part")) {
            NNTPPart *part = g_slice_new0(NNTPPart);
            xmlChar *partnum;

            partnum = xmlGetProp(cur, (const xmlChar *) "partnum");
            part->partnum = atoi((const char *) partnum);
            xmlFree(partnum);

            file->parts = g_list_append(file->parts, part);

            parse_part(doc, cur->xmlChildrenNode, file, part);
        }

        cur = cur->next;
    }

    file->numparts = g_list_length(file->parts);
}

static void
parse_file(xmlDocPtr doc, xmlNodePtr cur, NNTPCollection *collection, NNTPFile *file)
{
    // backwards compatibility with version 0.1 and 0.2
    file->num_parts_downloaded = -1;
    file->file_size_remaining = -1;

    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "Subject")) {
            xmlChar *subject = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (subject) {
                strncpy((char *) file->subject, (const char *) subject, (size_t) sizeof(file->subject));
                xmlFree(subject);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Poster")) {
            xmlChar *poster = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (poster) {
                strncpy((char *) file->poster, (const char *) poster, (size_t) sizeof(file->poster));
                xmlFree(poster);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Stamp")) {
            xmlChar *stamp = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (stamp) {
                file->stamp = atoi((const char *) stamp);
                xmlFree(stamp);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "FileSize")) {
            xmlChar *size = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (size) {
                file->file_size = atoll((const char *) size);
                xmlFree(size);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "NumPartsDownloaded")) {
            xmlChar *num_parts_downloaded = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (num_parts_downloaded) {
                file->num_parts_downloaded = atoi((const char *) num_parts_downloaded);
                xmlFree(num_parts_downloaded);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "FileIsDownloaded")) {
            xmlChar *file_is_downloaded = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (file_is_downloaded) {
                file->file_is_downloaded = atoi((const char *) file_is_downloaded);
                xmlFree(file_is_downloaded);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Groups")) {
            parse_groups(doc, cur->xmlChildrenNode, file);
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Parts")) {
            parse_parts(doc, cur->xmlChildrenNode, file);
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "TempFilename")) {
            xmlChar *tmp_filename = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (tmp_filename) {
                strncpy((char *) file->tmp_filename, (const char *) tmp_filename, (size_t) sizeof(file->tmp_filename));
                xmlFree(tmp_filename);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "RealFilename")) {
            xmlChar *real_filename = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (real_filename) {
                strncpy((char *) file->real_filename, (const char *) real_filename, (size_t) sizeof(file->real_filename));
                xmlFree(real_filename);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Status")) {
            xmlChar *status = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (status) {
                file->status = atoi((const char *) status);
                xmlFree(status);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "FileType")) {
            xmlChar *file_type = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (file_type) {
                file->file_type = atoi((const char *) file_type);
                xmlFree(file_type);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "PAR2StartNum")) {
            xmlChar *par2_startnum = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (par2_startnum) {
                file->par2_startnum = atoi((const char *) par2_startnum);
                xmlFree(par2_startnum);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "PAR2EndNum")) {
            xmlChar *par2_endnum = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (par2_endnum) {
                file->par2_endnum = atoi((const char *) par2_endnum);
                xmlFree(par2_endnum);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "PAR2NumBlocks")) {
            xmlChar *par2_numblocks = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (par2_numblocks) {
                file->num_par2_blocks = atoi((const char *) par2_numblocks);
                xmlFree(par2_numblocks);
            }
        }

        cur = cur->next;
    }

    // backwards compatibility (continued)
    if (file->num_parts_downloaded == -1) {
        file->num_parts_downloaded = g_list_length(file->parts);
    }

    if (file->file_is_downloaded) {
        file->num_parts_failed = g_list_length(file->parts) - file->num_parts_downloaded;
        file->file_size_remaining = 0;
    } else {
        GList *list = file->parts;

        file->file_size_remaining = 0;

        while (list) {
            NNTPPart *part = list->data;

            if (!part->downloaded) {
                file->file_size_remaining += part->size;
            }

            list = g_list_next(list);
        }
    }
}

static void
parse_files(xmlDocPtr doc, xmlNodePtr cur, NNTPCollection *collection)
{
    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "File")) {
            NNTPFile *file = g_slice_new0(NNTPFile);
            xmlChar *pos;

            collection->files = g_list_append(collection->files, file);

            pos = xmlGetProp(cur, (const xmlChar *) "position");
            if (pos) {
                file->position = atoi((const char *) pos);
                xmlFree(pos);
            } else {
                file->position = g_list_length(collection->files);
            }

            parse_file(doc, cur->xmlChildrenNode, collection, file);

            // Backwards compatibility
            if (file->par2_startnum == 0) {
                file->par2_startnum = -1;
            }

            if (file->par2_endnum == 0) {
                file->par2_endnum = -1;
            }
        }

        cur = cur->next;
    }
}

static NNTPCollection *
parse_collection(xmlDocPtr doc, xmlNodePtr cur)
{
    NNTPCollection *collection = g_slice_new0(NNTPCollection);
    GList *list;

    while(cur != NULL) {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "Name")) {
            xmlChar *name = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (name) {
                strncpy((char *) collection->collection_name, (const char *) name, (size_t) sizeof(collection->collection_name));
                xmlFree(name);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "TotalSize")) {
            xmlChar *size = xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
            if (size) {
                collection->total_size = atoll((const char *) size);
                xmlFree(size);
            }
        }

        if (!xmlStrcmp(cur->name, (const xmlChar *) "Files")) {
            parse_files(doc, cur->xmlChildrenNode, collection);
        }

        cur = cur->next;
    }

    // calculate the remaining size of the collection
    list = collection->files;

    collection->total_size_remaining = 0;
    while (list) {
        NNTPFile *file = list->data;

        collection->total_size_remaining += file->file_size_remaining;

        list = g_list_next(list);
    }

    return collection;
}

static gboolean
parse_xml_document(const char *contents, GList **ret, char **errmsg)
{
    xmlDocPtr    doc;
    xmlNodePtr   cur;

    g_assert(ret);

    *ret = NULL;

    xmlSetGenericErrorFunc(NULL, xml_parser_error_func);

    if (xml_parser_errmsg) {
        g_free(xml_parser_errmsg);
        xml_parser_errmsg = NULL;
    }

    doc = xmlParseMemory(contents, strlen(contents));
    if (doc == NULL) {
        if (errmsg) {
            if (xml_parser_errmsg) {
                *errmsg = g_strdup_printf(_("Invalid XML document:\n%s"), xml_parser_errmsg);
            } else {
                *errmsg = g_strdup(_("Invalid XML document"));
            }
        }

        g_free(xml_parser_errmsg);
        xml_parser_errmsg = NULL;

        xmlFreeDoc(doc);
        return FALSE;
    }

    if ((cur = xmlDocGetRootElement(doc)) && xmlStrcmp(cur->name, (const xmlChar *) "NNTPGrab")) {
        if (errmsg) {
            *errmsg = g_strdup("parse_xml_document: document of the wrong type, root node != NNTPGrab");
        }

        xmlFreeDoc(doc);
        return FALSE;
    }

    cur = cur->xmlChildrenNode;
    if (!cur) {
        if (errmsg) {
            *errmsg = g_strdup("parse_xml_document: no children of root-node!");
        }

        xmlFreeDoc(doc);
        return FALSE;
    }

    cur = cur->next;

    if (xmlStrcmp(cur->name, (const xmlChar *) "DownloadQueue")) {
        if (errmsg) {
            *errmsg = g_strdup("parse_xml_document: document of the wrong type, root node != NNTPGrab/DownloadQueue");
        }

        xmlFreeDoc(doc);
        return FALSE;
    }

    cur = cur->xmlChildrenNode;
    if (!cur) {
        // Nothing in the queue
        xmlFreeDoc(doc);
        return TRUE;
    }

    do {
        if (!xmlStrcmp(cur->name, (const xmlChar *) "Collection")) {
            NNTPCollection *collection = parse_collection(doc, cur->xmlChildrenNode);

            collection->position = g_list_length(*ret);

            *ret = g_list_append(*ret, collection);
        }
    } while ((cur=cur->next));

    xmlFreeDoc(doc);
    return TRUE;
}

gboolean
load_download_queue(GList **ret, char **errmsg)
{
    char *filename;
    char *contents = NULL;
    GError *err = NULL;
    gboolean retval;

    filename = generate_filename();
    if (!g_file_get_contents(filename, &contents, NULL, &err)) {
        // File not found, looks like this is the first time NNTPGrab is ever started, ignore
        g_error_free(err);
        g_free(filename);

        return TRUE;
    }

    retval = parse_xml_document(contents, ret, errmsg);

    g_free(filename);
    g_free(contents);

    return retval;
}

static void
save_part(xmlDocPtr doc, xmlNodePtr parent, NNTPPart *part)
{
    xmlNodePtr node, cdataNode;
    char *str;

    node = xmlNewNode(NULL, (const xmlChar *) "MessageID");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) part->message_id, strlen(part->message_id));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

    str = g_strdup_printf("%i", part->size);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "Size", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", part->downloaded);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "Downloaded", (const xmlChar *) str);
    g_free(str);
}

static void
save_file(xmlDocPtr doc, xmlNodePtr parent, NNTPFile *file)
{
    xmlNodePtr node, cdataNode, parent2;
    char *str;
    GList *list;

    if (file->file_size_remaining < 0) {
        if (file->file_is_downloaded) {
            file->file_size_remaining = 0;
        } else {
            GList *list = file->parts;

            file->file_size_remaining = 0;

            while (list) {
                NNTPPart *part = list->data;

                if (!part->downloaded) {
                    file->file_size_remaining += part->size;
                }

                list = g_list_next(list);
            }
        }
    }

    node = xmlNewNode(NULL, (const xmlChar *) "Subject");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) file->subject, strlen((const char *) file->subject));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

    node = xmlNewNode(NULL, (const xmlChar *) "Poster");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) file->poster, strlen((const char *) file->poster));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

    str = g_strdup_printf("%li", file->stamp);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "Stamp", (const xmlChar *) str);
    g_free(str);

#if SIZEOF_LONG == 8
    str = g_strdup_printf("%li", file->file_size);
#else /* SIZEOF_LONG == 4 */
    str = g_strdup_printf("%lli", file->file_size);
#endif
    xmlNewTextChild(parent, NULL, (const xmlChar *) "FileSize", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->file_is_downloaded);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "FileIsDownloaded", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->num_parts_downloaded);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "NumPartsDownloaded", (const xmlChar *) str);
    g_free(str);

    node = xmlNewNode(NULL, (const xmlChar *) "TempFilename");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) file->tmp_filename, strlen((const char *) file->tmp_filename));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

    node = xmlNewNode(NULL, (const xmlChar *) "RealFilename");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) file->real_filename, strlen((const char *) file->real_filename));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

    str = g_strdup_printf("%i", file->status);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "Status", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->file_type);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "FileType", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->par2_startnum);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "PAR2StartNum", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->par2_endnum);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "PAR2EndNum", (const xmlChar *) str);
    g_free(str);

    str = g_strdup_printf("%i", file->num_par2_blocks);
    xmlNewTextChild(parent, NULL, (const xmlChar *) "PAR2NumBlocks", (const xmlChar *) str);
    g_free(str);

    node = xmlNewNode(NULL, BAD_CAST "Groups");
    xmlAddChild(parent, node);

    parent2 = node;

    list = file->groups;
    while (list) {
        char *group = (char *) list->data;

        node = xmlNewNode(NULL, (const xmlChar *) "Group");
        cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) group, strlen(group));
        xmlAddChild(node, cdataNode);
        xmlAddChild(parent2, node);

        list = g_list_next(list);
    }

    node = xmlNewNode(NULL, BAD_CAST "Parts");
    xmlAddChild(parent, node);

    parent2 = node;

    list = file->parts;
    while (list) {
        NNTPPart *part = (NNTPPart *) list->data;

        node = xmlNewNode(NULL, BAD_CAST "Part");
        xmlAddChild(parent2, node);

        str = g_strdup_printf("%i", part->partnum);
        xmlNewProp(node, (const xmlChar *) "partnum", (const xmlChar *) str);
        g_free(str);

        save_part(doc, node, part);

        list = g_list_next(list);
    }
}

static void
save_collection(xmlDocPtr doc, xmlNodePtr parent, NNTPCollection *collection)
{
    xmlNodePtr node, cdataNode;
    char *str;
    GList *list;
    int i;

    node = xmlNewNode(NULL, (const xmlChar *) "Name");
    cdataNode = xmlNewCDataBlock(doc, (const xmlChar *) collection->collection_name, strlen(collection->collection_name));
    xmlAddChild(node, cdataNode);
    xmlAddChild(parent, node);

#if SIZEOF_LONG == 8
    str = g_strdup_printf("%li", collection->total_size);
#else /* SIZEOF_LONG == 4 */
    str = g_strdup_printf("%lli", collection->total_size);
#endif
    xmlNewTextChild(parent, NULL, (const xmlChar *) "TotalSize", (const xmlChar *) str);
    g_free(str);

    node = xmlNewNode(NULL, BAD_CAST "Files");
    xmlAddChild(parent, node);

    parent = node;

    list = collection->files;
    i = 0;
    while (list) {
        NNTPFile *file = (NNTPFile *) list->data;

        node = xmlNewNode(NULL, BAD_CAST "File");
        xmlAddChild(parent, node);

        i++;
        str = g_strdup_printf("%i", i);
        xmlNewProp(node, (const xmlChar *) "position", (const xmlChar *) str);
        g_free(str);

        save_file(doc, node, file);

        list = g_list_next(list);
    }
}

gboolean
save_download_queue(GList *queue, char **errmsg)
{
    xmlDocPtr doc;
    xmlNodePtr node, parent;
    GList *list;
    char *filename;
    char *dirname;

    doc = xmlNewDoc(BAD_CAST "1.0");
    node = xmlNewNode(NULL, BAD_CAST "NNTPGrab");
    xmlDocSetRootElement(doc, node);

    parent = node;
    node = xmlNewNode(NULL, BAD_CAST "DownloadQueue");
    xmlAddChild(parent, node);

    parent = node;

    list = queue;
    while (list) {
        NNTPCollection *collection = (NNTPCollection *) list->data;

        node = xmlNewNode(NULL, BAD_CAST "Collection");
        xmlAddChild(parent, node);

        save_collection(doc, node, collection);

        list = g_list_next(list);
    }

    // Generate the folder ~/.config/NNTPGrab in case it didn't exist yet
    filename = generate_filename();
    dirname = g_path_get_dirname(filename);
    g_mkdir_with_parents(dirname, 0700);
    g_free(dirname);

    if (xmlSaveFormatFile(filename, doc, 1) == -1) {
        // TODO: Proper error message
        if (errmsg) {
            *errmsg = g_strdup_printf(_("Error while saving XML document '%s'"), filename);
        }

        g_free(filename);
        xmlFreeDoc(doc);

        return FALSE;
    }

    g_free(filename);
    xmlFreeDoc(doc);

    return TRUE;
}

void
free_download_queue(GList *queue)
{
    GList *list;

    list = queue;
    while (list) {
        GList *list2;
        NNTPCollection *collection = (NNTPCollection *) list->data;

        list2 = collection->files;
        while (list2) {
            GList *list3;
            NNTPFile *file = (NNTPFile *) list2->data;

            list3 = file->groups;
            while (list3) {
                g_free(list3->data);
                list3 = g_list_next(list3);
            }

            g_list_free(file->groups);

            list3 = file->parts;
            while (list3) {
                g_slice_free(NNTPPart, list3->data);
                list3 = g_list_next(list3);
            }

            g_list_free(file->parts);

            g_slice_free(NNTPFile, file);

            list2 = g_list_next(list2);
        }

        g_slice_free(NNTPCollection, collection);

        list = g_list_next(list);
    }

    g_list_free(queue);
}

---