NNTPGrab

/* 

    Copyright (C) 2005-2010  Erik van Pienbroek

    This program is free software; you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by

    the Free Software Foundation; either version 2 of the License, or

    (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License

    along with this program; if not, write to the Free Software

    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

#include 

#include 

#include 

#include 

#include 

#include 

#include "nntpgrab_plugin.h"

#include "nntpgrab_utils.h"

#include "marshalers.h"

#include "config.h"

#if GLIB_CHECK_VERSION(2,16,0)

#include "fr-archive.h"

#include "fr-command.h"

#include "fr-mime.h"

typedef struct _plugin_unpack_priv {

    const char *collection_name;

    const char *filename;

    GThreadPool *thread_pool;

} PluginUnpackPriv;

struct _unpack_data {

    NGPlugin *plugin_data;

    FrArchive *archive;

    gboolean now_loading;

    GMutex *mutex;

    GCond *cond;

    gboolean error_occured;

    FrProcErrorType error_type;

    char errmsg[1024];

    char filename[1024];

    char target_directory[1024];

    GList *files_extracted;

};

static gboolean unpack_do_unpack(NGPlugin *plugin_data, const char *collection_name, const char *filename, const char *target_directory, char **errmsg);

static void perform_automatic_unpack(gpointer data, gpointer user_data);

static void on_collection_downloaded(NGPlugin *plugin_data, const char *collection_name, gpointer data);

static void on_par2_repair_success(NGPlugin *plugin_data, const char *plugin_name, const char **params);

static void on_par2_no_repair_required(NGPlugin *plugin_data, const char *plugin_name, const char **params);

#endif

void

nntpgrab_plugin_initialize(NGPlugin *plugin_data)

{

    ng_plugin_set_name(plugin_data, "Unpack");

    ng_plugin_set_version(plugin_data, PACKAGE_VERSION);

    ng_plugin_set_author(plugin_data, "Erik van Pienbroek");

    ng_plugin_set_url(plugin_data, "https://www.nntpgrab.nl");

    ng_plugin_set_description(plugin_data, "This plugin is responsible for unpacking files after they've been completely downloaded");

#if GLIB_CHECK_VERSION(2,16,0)

    /* Register all the functions which are implemented by this plugin */

    ng_plugin_register_function(plugin_data,

                                "unpack_do_unpack",

                                NG_PLUGIN_FUNCTION(unpack_do_unpack),

                                ng_plugin_marshal_BOOLEAN__STRING_STRING_STRING_POINTER,

                                G_TYPE_BOOLEAN,

                                4, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_POINTER);

    /* Indicate the events which we can emit */

    ng_plugin_create_event(plugin_data, "unpack_progress_update", 3);

    ng_plugin_create_event(plugin_data, "unpack_message_received", 3);

    ng_plugin_create_event(plugin_data, "unpack_working_archive_changed", 3);

    ng_plugin_create_event(plugin_data, "unpack_success", 2);

    ng_plugin_create_event(plugin_data, "unpack_failure", 3);

    /* Add a dependency on the NNTPGrab Core and the PAR2 plugin */

    ng_plugin_set_required_event(plugin_data, "collection_downloaded");

    ng_plugin_set_required_event(plugin_data, "par2_repair_success");

    ng_plugin_set_required_event(plugin_data, "par2_no_repair_required");

#endif

}

ngboolean

nntpgrab_plugin_load(NGPlugin *plugin_data, char **errmsg)

{

    GError *err = NULL;

#if GLIB_CHECK_VERSION(2,16,0)

    plugin_data->priv = g_slice_new0(PluginUnpackPriv);

    ((PluginUnpackPriv *) plugin_data->priv)->thread_pool = g_thread_pool_new(perform_automatic_unpack, plugin_data, 1, FALSE, &err);

    if (!((PluginUnpackPriv *) plugin_data->priv)->thread_pool) {

        *errmsg = g_strdup_printf(_("%s:%i Unable to create Unpack thread pool: %s"), __FILE__, __LINE__, err->message);

        g_error_free(err);

        return FALSE;

    }

    ng_plugin_connect_event(plugin_data, "collection_downloaded", NG_PLUGIN_FUNCTION(on_collection_downloaded), NULL);

    ng_plugin_connect_event(plugin_data, "par2_repair_success", NG_PLUGIN_FUNCTION(on_par2_repair_success), NULL);

    ng_plugin_connect_event(plugin_data, "par2_no_repair_required", NG_PLUGIN_FUNCTION(on_par2_no_repair_required), NULL);

    fr_command_register_commands();

    return TRUE;

#else

    /* GLib version too old */

    return TRUE;

#endif

}

ngboolean

nntpgrab_plugin_can_unload(NGPlugin *plugin_data, char **reason)

{

    return TRUE;

}

void

nntpgrab_plugin_unload(NGPlugin *plugin_data)

{

#if GLIB_CHECK_VERSION(2,16,0)

    /* TODO: fr_command_unregister_commands() */

    g_thread_pool_free(((PluginUnpackPriv *) plugin_data->priv)->thread_pool, TRUE, FALSE);

    g_slice_free(PluginUnpackPriv, plugin_data->priv);

    plugin_data->priv = NULL;

#endif

}

void

nntpgrab_plugin_destroy(NGPlugin *plugin_data)

{

}

int

nntpgrab_plugin_get_version(void)

{

    return NNTPGRAB_PLUGIN_API_VERSION;

}

#if GLIB_CHECK_VERSION(2,16,0)

static void

on_start(FrArchive *archive, FrAction action, gpointer data)

{

g_print("start: action = %i\n", action);

}

static void

on_message(FrArchive *archive, const char *message, gpointer data)

{

    const char *params[4];

    struct _unpack_data *unpack_data = data;

    PluginUnpackPriv *priv = unpack_data->plugin_data->priv;

    params[0] = priv->collection_name;

    params[1] = priv->filename;

    params[2] = message;

    params[3] = NULL;

    ng_plugin_emit_event(unpack_data->plugin_data, "unpack_message_received", params);

}

static void

on_progress(FrArchive *archive, double progress, gpointer data)

{

    const char *params[4];

    char progress_str[16];

    struct _unpack_data *unpack_data = data;

    PluginUnpackPriv *priv = unpack_data->plugin_data->priv;

    memset(&progress_str, 0, sizeof(progress_str));

    snprintf(progress_str, sizeof(progress_str) - 1, "%.2f", progress);

    params[0] = priv->collection_name;

    params[1] = priv->filename;

    params[2] = progress_str;

    params[3] = NULL;

    ng_plugin_emit_event(unpack_data->plugin_data, "unpack_progress_update", params);

}

static void

on_working_archive(FrArchive *archive, const char *filename, gpointer data)

{

    const char *params[4];

    struct _unpack_data *unpack_data = data;

    char *filename_local = g_filename_from_uri(filename, NULL, NULL);

    char *filename_short;

    PluginUnpackPriv *priv = unpack_data->plugin_data->priv;

    params[0] = priv->collection_name;

    params[1] = priv->filename;

    params[3] = NULL;

    if (filename_local) {

        /* Maintain a list of all extracted files so we can remove them afterwards */

        unpack_data->files_extracted = g_list_append(unpack_data->files_extracted, g_strdup(filename_local));

        filename_short = g_path_get_basename(filename_local);

        params[2] = filename_short;

        ng_plugin_emit_event(unpack_data->plugin_data, "unpack_working_archive_changed", params);

    } else {

        filename_short = g_path_get_basename(filename);

        params[2] = filename_short;

        ng_plugin_emit_event(unpack_data->plugin_data, "unpack_working_archive_changed", params);

    }

    g_free(filename_short);

    g_free(filename_local);

}

static gboolean

do_start_unpack(gpointer data)

{

    struct _unpack_data *unpack_data = data;

    ng_plugin_emit_log_msg(unpack_data->plugin_data, NG_LOG_LEVEL_DEBUG, "Now starting with extract phase");

    unpack_data->now_loading = FALSE;

    fr_process_clear (unpack_data->archive->process);

    fr_archive_extract_to_local(unpack_data->archive, NULL, unpack_data->target_directory, NULL, FALSE, TRUE, FALSE, NULL);

    fr_process_start (unpack_data->archive->process);

    return FALSE;

}

static void

on_done(FrArchive *archive, FrAction action, FrProcError *error, gpointer data)

{

    /* Signal the unpack thread that we're finished */

    struct _unpack_data *unpack_data = data;

    unpack_data->error_type = error->type;

    unpack_data->error_occured = (error->status != 0 || error->type != 0);

    if (error->gerror && error->gerror->message && strlen(error->gerror->message) > 0) {

        strncpy(unpack_data->errmsg, error->gerror->message, sizeof(unpack_data->errmsg) - 1);

    } else if (error->status != 0) {

        snprintf(unpack_data->errmsg, sizeof(unpack_data->errmsg) - 1, "Unknown error occured while unpacking, error->status = %i, error->type = %i", error->status, error->type);

    }

    if (action == FR_ACTION_LISTING_CONTENT && !unpack_data->error_occured) {

        g_idle_add(do_start_unpack, unpack_data);

    }

    if (action == FR_ACTION_EXTRACTING_FILES || unpack_data->error_occured) {

        on_progress(archive, 1.0, unpack_data);

        if (unpack_data->error_occured) {

            on_message(archive, unpack_data->errmsg, unpack_data);

        }

        /* Send a signal to the unpack thread that we're done */

        g_mutex_lock(unpack_data->mutex);

        g_cond_signal(unpack_data->cond);

        g_mutex_unlock(unpack_data->mutex);

    }

}

static gboolean

unpack_do_unpack(NGPlugin *plugin_data, const char *collection_name, const char *filename, const char *target_directory, char **errmsg)

{

    struct _unpack_data *unpack_data;

    FrArchive *archive = fr_archive_new();

    gboolean ret;

    char *filename_uri;

    char *filename_short;

    GError *err = NULL;

    PluginUnpackPriv *priv = plugin_data->priv;

    GList *list;

    NGConfigOpts opts;

    filename_uri = g_filename_to_uri(filename, NULL, &err);

    if (!filename_uri) {

        if (errmsg) {

            *errmsg = g_strdup_printf(_("Unable to convert filename to URI: %s (filename = %s)"), err->message, filename);

        } else {

            g_warning(_("Unable to convert filename to URI: %s (filename = %s)"), err->message, filename);

        }

        g_error_free(err);

        return FALSE;

    }

    priv->collection_name = collection_name;

    priv->filename = filename;

    /* The File Roller code is heavily integrated in the GLib mainloop 

     * As this is a seperate thread, we just have to wait here until

     * we have received a signal that the extracting has completed */

    unpack_data = g_slice_new0(struct _unpack_data);

    unpack_data->plugin_data = plugin_data;

    unpack_data->cond = g_cond_new();

    unpack_data->mutex = g_mutex_new();

    unpack_data->archive = archive;

    unpack_data->files_extracted = NULL;

    filename_short = g_path_get_basename(filename);

    strncpy(unpack_data->filename, filename_short, sizeof(unpack_data->filename) - 1);

    g_free(filename_short);

    strncpy(unpack_data->target_directory, target_directory, sizeof(unpack_data->target_directory) - 1);

    g_signal_connect(archive, "start", G_CALLBACK(on_start), unpack_data);

    g_signal_connect(archive, "done", G_CALLBACK(on_done), unpack_data);

    g_signal_connect(archive, "progress", G_CALLBACK(on_progress), unpack_data);

    g_signal_connect(archive, "working_archive", G_CALLBACK(on_working_archive), unpack_data);

    g_signal_connect(archive, "message", G_CALLBACK(on_message), unpack_data);

    if (!fr_archive_load_local(archive, filename_uri, NULL)) {

        if (errmsg) {

            *errmsg = g_strdup_printf("Unable to load files to unpack");

        }

        g_object_unref(archive);

        g_slice_free(struct _unpack_data, unpack_data);

        g_free(filename_uri);

        priv->collection_name = NULL;

        priv->filename = NULL;

        return FALSE;

    }

    /* Wait for a confirmation that extraction was successfull */

    g_mutex_lock(unpack_data->mutex);

    g_cond_wait(unpack_data->cond, unpack_data->mutex);

    g_mutex_unlock(unpack_data->mutex);

    ret = !unpack_data->error_occured;

    if (!ret && errmsg) {

        *errmsg = g_strdup(unpack_data->errmsg);

    }

    /* Add the archive itself to the list of files which need to be removed. 

     * For .rar archives spanning multiple files this will be done automatically

     * in the 'working_archive_signal' signal, but this signal doesn't occur

     * for single file archives */

    unpack_data->files_extracted = g_list_append(unpack_data->files_extracted, g_strdup(filename));

    /* Automatically remove the archives if the users wants this */

    opts = plugin_data->core_funcs.config_get_opts();

    list = unpack_data->files_extracted;

    while (list) {

        char *path = list->data;

        if (ret && opts.auto_remove_files) {

            ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, _("Now automatically removing file '%s'"), path);

            g_unlink(path);

        }

        g_free(path);

        list = g_list_next(list);

    }

    g_list_free(unpack_data->files_extracted);

    g_mutex_free(unpack_data->mutex);

    g_cond_free(unpack_data->cond);

    g_object_unref(archive);

    g_slice_free(struct _unpack_data, unpack_data);

    g_free(filename_uri);

    priv->collection_name = NULL;

    priv->filename = NULL;

    return ret;

}

typedef struct _unpack_file {

    char filename_without_extension[256];

    char filename[256];

} UnpackFile;

struct _unpack_thread_data {

    NGPlugin *plugin_data;

    char collection_name[256];

    NGConfigOpts opts;

};

static void

perform_automatic_unpack(gpointer data, gpointer user_data)

{

    struct _unpack_thread_data *thread_data = (struct _unpack_thread_data *) data;

    static NGRegex *re = NULL;

    const char *filename;

    char *target_directory;

    char *errmsg = NULL;

    GError *err= NULL;

    GDir *dir;

    GList *list;

    GList *files = NULL;

    NGPlugin *plugin_data;

    const char *collection_name;

    NGConfigOpts opts;

    static GStaticMutex mutex = G_STATIC_MUTEX_INIT;

    g_static_mutex_lock(&mutex);

    if (!re) {

        re = nntpgrab_utils_regex_compile(".*\\.part(\\d+)\\.rar$");

    }

    plugin_data = thread_data->plugin_data;

    collection_name = thread_data->collection_name;

    opts = thread_data->opts;

    target_directory = g_build_filename(opts.download_directory, collection_name, NULL);

    dir = g_dir_open(target_directory, 0, &err);

    if (!dir) {

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_WARNING, _("Unable to open directory '%s': %s"), target_directory, err->message);

        g_error_free(err);

        g_free(target_directory);

        g_slice_free(struct _unpack_thread_data, thread_data);

        g_static_mutex_unlock(&mutex);

        return;

    }

    while ((filename = g_dir_read_name(dir))) {

        /* Is this a file in the notation '.partXXX.rar' ? */

        char *filename_lowercase = g_ascii_strdown(filename, -1);

        const char **matches;

        if ((matches = nntpgrab_utils_regex_match(re, filename_lowercase))) {

            /* Is this .part001.rar ? */

            if (matches[0] && matches[1] && atoi(matches[1]) == 1) {

                UnpackFile *file = g_slice_new0(UnpackFile);

                strncpy(file->filename, filename, sizeof(file->filename) - 1);

                files = g_list_append(files, file);

            }

            nntpgrab_utils_regex_matches_free(matches);

        } else if (!strcmp(filename_lowercase + strlen(filename_lowercase) - 4, ".rar")) {

            /* A .rar file detected which isn't part of a .partXX.rar set */

            UnpackFile *file = g_slice_new0(UnpackFile);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            files = g_list_append(files, file);

        } else if (!strcmp(filename_lowercase + strlen(filename_lowercase) - 4, ".zip")) {

            /* A .zip file detected */

            UnpackFile *file = g_slice_new0(UnpackFile);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            files = g_list_append(files, file);

        } else if (!strcmp(filename_lowercase + strlen(filename_lowercase) -4, ".001")) {

            /* Some .001 files can be rar archives, check the mime type for such files */

            GFile *file;

            GFileInfo *info;

            char *path = g_build_filename(target_directory, filename, NULL);

            file = g_file_new_for_path(path);

            info = g_file_query_info(file, "standard::content-type", G_FILE_QUERY_INFO_NONE, NULL, NULL);

            g_free(path);

            if (info) {

                if (!strcmp(g_file_info_get_content_type(info), "application/x-rar")) {

                    UnpackFile *file = g_slice_new0(UnpackFile);

                    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, "A .001 file was detected which actually is an rar archive: %s", filename);

                    strncpy(file->filename, filename, sizeof(file->filename) - 1);

                    files = g_list_append(files, file);

                } else {

                    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, "A .001 file was detected, but it isn't an rar archive, ignoring: %s", filename);

                }

                g_object_unref(info);

            }

            g_object_unref(file);

#if 0 

        } else {

            g_print(__FILE__ ":%i file ignored\n\n", __LINE__);

#endif

        }

        g_free(filename_lowercase);

    }

    g_dir_close(dir);

    if (!files) {

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, _("No files found to unpack for collection '%s'"), collection_name);

    }

    /* Perform the unpack */

    list = files;

    while (list) {

        UnpackFile *file = (UnpackFile*) list->data;

        char *total_filename = g_build_filename(target_directory, file->filename, NULL);

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Now unpacking '%s'"), file->filename);

        if (!unpack_do_unpack(plugin_data, collection_name, total_filename, target_directory, &errmsg)) {

            const char *params[4];

            if (!errmsg || strlen(errmsg) == 0) {

                errmsg = g_strdup_printf(_("process stopped unexpectedly"));

            }

            params[0] = collection_name;

            params[1] = file->filename;

            params[2] = errmsg;

            params[3] = NULL;

            ng_plugin_emit_event(plugin_data, "unpack_failure", params);

            ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_WARNING, _("Automatic unpack failed due to an error: %s"), errmsg);

            g_free(errmsg);

        } else {

            const char *params[3];

            params[0] = collection_name;

            params[1] = file->filename;

            params[2] = NULL;

            ng_plugin_emit_event(plugin_data, "unpack_success", params);

        }

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Unpack of '%s' finished"), file->filename);

        g_free(total_filename);

        g_slice_free(UnpackFile, file);

        list = g_list_next(list);

    }

    g_list_free(files);

    g_free(target_directory);

    g_slice_free(struct _unpack_thread_data, thread_data);

    g_static_mutex_unlock(&mutex);

}

static gboolean

check_if_folder_contains_par2_files(NGPlugin *plugin_data, const char *collection_name)

{

    NGConfigOpts opts;

    char *directory;

    GDir *dir;

    GError *err = NULL;

    const char *filename;

    opts = plugin_data->core_funcs.config_get_opts();

    directory = g_build_filename(opts.download_directory, collection_name, NULL);

    dir = g_dir_open(directory, 0, &err);

    if (!dir) {

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_WARNING, _("Unable to open directory '%s': %s"), directory, err->message);

        g_error_free(err);

        g_free(directory);

        return FALSE;

    }

    while ((filename = g_dir_read_name(dir))) {

        char *extension;

        extension = g_strrstr(filename, ".");

        if (!extension) {

            /* File without extension, ignore */

            continue;

        }

        if (g_ascii_strncasecmp(extension, ".par", 4) != 0) {

            /* Different extension, ignore */

            continue;

        }

        /* File with .par or .par2 extension found */

        g_dir_close(dir);

        return TRUE;

    }

    g_dir_close(dir);

    return FALSE;

}

static void

on_collection_downloaded(NGPlugin *plugin_data, const char *collection_name, gpointer data)

{

    struct _unpack_thread_data *thread_data;

    NGConfigOpts opts;

    /* Only try to unpack files here when automatic PAR2 repair is disabled */

    opts = plugin_data->core_funcs.config_get_opts();

    if (!opts.enable_auto_unpack || plugin_data->core_funcs.schedular_get_state() != SCHEDULAR_STATE_RUNNING ||

        /* If the collection doesn't contain any par2 files, unpack directly */

        (opts.enable_par2_repair && check_if_folder_contains_par2_files(plugin_data, collection_name))) {

        /* Come back after the PAR2 repair has completed */

        return;

    }

    thread_data = g_slice_new0(struct _unpack_thread_data);

    thread_data->plugin_data = plugin_data;

    strncpy(thread_data->collection_name, collection_name, sizeof(thread_data->collection_name) - 1);

    thread_data->opts = opts;

    g_thread_pool_push(((PluginUnpackPriv *) plugin_data->priv)->thread_pool, thread_data, NULL);

}

static void

on_par2_repair_success(NGPlugin *plugin_data, const char *plugin_name, const char **params)

{

    struct _unpack_thread_data *thread_data;

    NGConfigOpts opts;

    if (!strcmp(params[2], "TRUE")) {

        /* This collection still has more PAR2 sets. Try again later when all PAR2 sets in this collection have been verified */

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Ignoring automatic unpack for collection '%s' for now because there are more PAR2 sets to process first"), params[0]);

        return;

    }

    /* Only try to unpack files here when the schedular is still running */

    opts = plugin_data->core_funcs.config_get_opts();

    if (!opts.enable_auto_unpack || plugin_data->core_funcs.schedular_get_state() != SCHEDULAR_STATE_RUNNING) {

        return;

    }

    thread_data = g_slice_new0(struct _unpack_thread_data);

    thread_data->plugin_data = plugin_data;

    strncpy(thread_data->collection_name, params[0], sizeof(thread_data->collection_name) - 1);

    thread_data->opts = opts;

    g_thread_pool_push(((PluginUnpackPriv *) plugin_data->priv)->thread_pool, thread_data, NULL);

}

static void

on_par2_no_repair_required(NGPlugin *plugin_data, const char *plugin_name, const char **params)

{

    struct _unpack_thread_data *thread_data;

    NGConfigOpts opts;

    if (!strcmp(params[2], "TRUE")) {

        /* This collection still has more PAR2 sets. Try again later when all PAR2 sets in this collection have been verified */

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Ignoring automatic unpack for collection '%s' for now because there are more PAR2 sets to process first"), params[0]);

        return;

    }

    /* Only try to unpack files here when the schedular is still running */

    opts = plugin_data->core_funcs.config_get_opts();

    if (!opts.enable_auto_unpack ||plugin_data->core_funcs.schedular_get_state() != SCHEDULAR_STATE_RUNNING) {

        return;

    }

    thread_data = g_slice_new0(struct _unpack_thread_data);

    thread_data->plugin_data = plugin_data;

    strncpy(thread_data->collection_name, params[0], sizeof(thread_data->collection_name) - 1);

    thread_data->opts = opts;

    g_thread_pool_push(((PluginUnpackPriv *) plugin_data->priv)->thread_pool, thread_data, NULL);

}

#endif