NNTPGrab

/* 

    Copyright (C) 2005-2010  Erik van Pienbroek

    This program is free software; you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by

    the Free Software Foundation; either version 2 of the License, or

    (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License

    along with this program; if not, write to the Free Software

    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

#include 

#include 

#include 

#include 

#include 

#include 

#include 

#include 

#ifndef WIN32

#include 

#endif

#include "nntpgrab_plugin.h"

#include "marshalers.h"

#include "config.h"

static gboolean nntpgrab_plugin_par2_repair_files(NGPlugin *plugin_data, const char *collection_name, const char *directory, const char *par2filename, ngboolean more_par2_sets_remaining_in_collection);

static void on_collection_downloaded(NGPlugin *plugin_data, const char *collection_name, gpointer data);

static void par2_thread_func(gpointer data, gpointer user_data);

typedef struct _plugin_par2_priv {

    GThreadPool *thread_pool;

    const char *collection_name;

    const char *par2filename;

    GList *par2files;

    gboolean repair_succeeded;

    gboolean command_completed_normally;

    gboolean abort_flag;

    int pid;

} PluginPAR2Priv;

#ifdef WIN32

#include 

#include 

#undef popen

#undef pclose

#define popen pt_popen

#define pclose pt_pclose

HANDLE my_pipein[2], my_pipeout[2], my_pipeerr[2];

char   my_popenmode = ' ';

static int

my_pipe(HANDLE *readwrite)

{

    SECURITY_ATTRIBUTES sa;

    sa.nLength = sizeof(sa);

    sa.bInheritHandle = 1;

    sa.lpSecurityDescriptor = NULL;

    if (!CreatePipe (&readwrite[0],&readwrite[1],&sa,1 << 13)) {

        errno = EMFILE;

        return -1;

    }

    return 0;

}

static FILE *

pt_popen(const char *cmd, const char *mode)

{

    FILE *fptr = (FILE *)0;

    PROCESS_INFORMATION piProcInfo;

    STARTUPINFO siStartInfo;

    int success, catch_stderr;

    my_pipein[0]   = INVALID_HANDLE_VALUE;

    my_pipein[1]   = INVALID_HANDLE_VALUE;

    my_pipeout[0]  = INVALID_HANDLE_VALUE;

    my_pipeout[1]  = INVALID_HANDLE_VALUE;

    my_pipeerr[0]  = INVALID_HANDLE_VALUE;

    my_pipeerr[1]  = INVALID_HANDLE_VALUE;

    if (!mode || !*mode)

        goto finito;

    my_popenmode = *mode;

    if (my_popenmode != 'r' && my_popenmode != 'w')

        goto finito;

    catch_stderr = strstr((char *) cmd, "2>&1") != NULL;

    if (catch_stderr) {

        char *ptr = strstr((char*) cmd, "2>&1");

        *ptr = '\0';

    }

    if (my_pipe(my_pipein)  == -1 || my_pipe(my_pipeout) == -1)

        goto finito;

    if (!catch_stderr && my_pipe(my_pipeerr) == -1)

      goto finito;

    ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));

    siStartInfo.cb             = sizeof(STARTUPINFO);

    siStartInfo.hStdInput      = my_pipein[0];

    siStartInfo.hStdOutput     = my_pipeout[1];

    if (catch_stderr)

        siStartInfo.hStdError  = my_pipeout[1];

    else

        siStartInfo.hStdError  = my_pipeerr[1];

    siStartInfo.dwFlags        = STARTF_USESTDHANDLES;

    success = CreateProcess(NULL,

       (LPTSTR)cmd,       // command line

       NULL,              // process security attributes

       NULL,              // primary thread security attributes

       TRUE,              // handles are inherited

       DETACHED_PROCESS,  // creation flags

       NULL,              // use parent's environment

       NULL,              // use parent's current directory

       &siStartInfo,      // STARTUPINFO pointer

       &piProcInfo);      // receives PROCESS_INFORMATION

    if (!success)

        goto finito;

    CloseHandle(my_pipein[0]);  my_pipein[0]  = INVALID_HANDLE_VALUE;

    CloseHandle(my_pipeout[1]); my_pipeout[1] = INVALID_HANDLE_VALUE;

    CloseHandle(my_pipeerr[1]); my_pipeerr[1] = INVALID_HANDLE_VALUE;

    if (my_popenmode == 'r')

        fptr = _fdopen(_open_osfhandle((intptr_t)my_pipeout[0],_O_BINARY),"r");

    else

        fptr = _fdopen(_open_osfhandle((intptr_t)my_pipein[1],_O_BINARY),"w");

finito:

    if (!fptr) {

        if (my_pipein[0]  != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipein[0]);

        if (my_pipein[1]  != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipein[1]);

        if (my_pipeout[0] != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipeout[0]);

        if (my_pipeout[1] != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipeout[1]);

        if (my_pipeerr[0] != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipeerr[0]);

        if (my_pipeerr[1] != INVALID_HANDLE_VALUE)

            CloseHandle(my_pipeerr[1]);

    }

    return fptr;

}

static int

pt_pclose(FILE *file)

{

    if (!file) {

        return -1;

    }

    fclose(file);

    CloseHandle(my_pipeerr[0]);

    if (my_popenmode == 'r')

        CloseHandle(my_pipein[1]);

    else

       CloseHandle(my_pipeout[0]);

    return 0;

}

#endif

void

nntpgrab_plugin_initialize(NGPlugin *plugin_data)

{

    ng_plugin_set_name(plugin_data, "PAR2");

    ng_plugin_set_version(plugin_data, PACKAGE_VERSION);

    ng_plugin_set_author(plugin_data, "Erik van Pienbroek");

    ng_plugin_set_url(plugin_data, "https://www.nntpgrab.nl");

    ng_plugin_set_description(plugin_data, "Automatically verify and repair files using PAR2 after a collection has been downloaded");

    ng_plugin_set_required_event(plugin_data, "collection_downloaded");

    ng_plugin_register_function(plugin_data,

                                "par2_repair_files",

                                NG_PLUGIN_FUNCTION(nntpgrab_plugin_par2_repair_files),

                                ng_plugin_marshal_BOOLEAN__STRING_STRING_STRING,

                                G_TYPE_BOOLEAN,

                                3, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING);

    ng_plugin_create_event(plugin_data, "par2_begin_verify", 2);

    ng_plugin_create_event(plugin_data, "par2_load_progress_update", 4);

    ng_plugin_create_event(plugin_data, "par2_recovery_file_loaded", 5);

    ng_plugin_create_event(plugin_data, "par2_file_loaded", 6);

    ng_plugin_create_event(plugin_data, "par2_repair_progress_update", 3);

    ng_plugin_create_event(plugin_data, "par2_repair_failure", 4);

    ng_plugin_create_event(plugin_data, "par2_repair_success", 3);

    ng_plugin_create_event(plugin_data, "par2_no_repair_required", 3);

}

ngboolean

nntpgrab_plugin_load(NGPlugin *plugin_data, char **errmsg)

{

    GError *err = NULL;

    PluginPAR2Priv *priv;

    plugin_data->priv = g_slice_new0(PluginPAR2Priv);

    priv = plugin_data->priv;

    priv->abort_flag = FALSE;

    priv->thread_pool = g_thread_pool_new(par2_thread_func, plugin_data, 1, FALSE, &err);

    if (!priv->thread_pool) {

        *errmsg = g_strdup_printf(_("%s:%i Unable to create PAR2 thread pool: %s"), __FILE__, __LINE__, err->message);

        g_error_free(err);

        return FALSE;

    }

    ng_plugin_connect_event(plugin_data, "collection_downloaded", NG_PLUGIN_FUNCTION(on_collection_downloaded), NULL);

    return TRUE;

}

ngboolean

nntpgrab_plugin_can_unload(NGPlugin *plugin_data, char **reason)

{

    return TRUE;

}

void

nntpgrab_plugin_unload(NGPlugin *plugin_data)

{

    PluginPAR2Priv *priv = plugin_data->priv;

    priv->abort_flag = TRUE;

#ifndef WIN32

    if (priv->pid > 0) {

        kill(priv->pid, SIGTERM);

    }

#endif

    g_thread_pool_free(priv->thread_pool, TRUE, TRUE);

    g_slice_free(PluginPAR2Priv, plugin_data->priv);

    plugin_data->priv = NULL;

}

void

nntpgrab_plugin_destroy(NGPlugin *plugin_data)

{

}

int

nntpgrab_plugin_get_version(void)

{

    return NNTPGRAB_PLUGIN_API_VERSION;

}

#if 0 

static bool

LoadVerificationPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header)

{

  VerificationPacket *packet = new VerificationPacket;

  // Load the packet from disk

  if (!packet->Load(diskfile, offset, header))

  {

    delete packet;

    return false;

  }

  // What is the fileid

  const MD5Hash &fileid = packet->FileId();

  // Look up the fileid in the source file map for an existing source file entry

  map::iterator sfmi = sourcefilemap.find(fileid);

  Par2RepairerSourceFile *sourcefile = (sfmi == sourcefilemap.end()) ? 0 :sfmi->second;

  // Was there an existing source file

  if (sourcefile)

  {

    // Does the source file already have a verification packet

    if (sourcefile->GetVerificationPacket())

    {

      // Yes. We don't need another copy.

      delete packet;

      return false;

    }

    else

    {

      // No. Store the packet in the source file

      sourcefile->SetVerificationPacket(packet);

      return true;

    }

  }

  else

  {

    // Create a new source file for the packet

    sourcefile = new Par2RepairerSourceFile(NULL, packet);

    // Record the source file in the source file map

    sourcefilemap.insert(pair(fileid, sourcefile));

    return true;

  }

}

#endif

gboolean

nntpgrab_plugin_par2_load_verification_packet(const char *filename, PAR2Set **par2set)

{

#if 0 

    DiskFile *file;

    VerificationPacket *packet = NULL;

    g_return_val_if_fail(filename != NULL, FALSE);

    g_return_val_if_fail(par2set != NULL, FALSE);

    *par2set = NULL;

    file = new DiskFile();

    if (!file->Open(filename)) {

        delete file;

        return FALSE;

    }

    g_print("Now reading '%s' for PAR2 verification data..\n", filename);

    // How many useable packets have we found

    uint32_t packets = 0;

    // How big is the file

    guint64 filesize = file->FileSize();

    if (filesize <= 0) {

        delete file;

        g_print("Filesize == 0\n");

        return FALSE;

    }

    // Allocate a buffer to read data into

    // The buffer should be large enough to hold a whole

    // critical packet (i.e. file verification, file description, main,

    // and creator), but not necessarily a whole recovery packet.

    size_t buffersize = (size_t)min((u64)1048576, filesize);

    u8 *buffer = new u8[buffersize];

    // Start at the beginning of the file

    guint64 offset = 0;

    // Continue as long as there is at least enough for the packet header

    while (offset + sizeof(PACKET_HEADER) <= filesize) {

        // Attempt to read the next packet header

        PACKET_HEADER header;

        if (!file->Read(offset, &header, sizeof(header)))

            break;

        // Does this look like it might be a packet

        if (packet_magic != header.magic) {

            offset++;

            // Is there still enough for at least a whole packet header

            while (offset + sizeof(PACKET_HEADER) <= filesize) {

                // How much can we read into the buffer

                size_t want = (size_t)min((u64)buffersize, filesize-offset);

                // Fill the buffer

                if (!file->Read(offset, buffer, want)) {

                    offset = filesize;

                    break;

                }

                // Scan the buffer for the magic value

                u8 *current = buffer;

                u8 *limit = &buffer[want-sizeof(PACKET_HEADER)];

                while (current <= limit && packet_magic != ((PACKET_HEADER*)current)->magic) {

                    current++;

                }

                // What file offset did we reach

                offset += current-buffer;

                // Did we find the magic

                if (current <= limit) {

                    memcpy(&header, current, sizeof(header));

                    break;

                }

            }

            // Did we reach the end of the file

            if (offset + sizeof(PACKET_HEADER) > filesize)

            {

                break;

            }

        }

        // We have found the magic

        // Check the packet length

        if (sizeof(PACKET_HEADER) > header.length || // packet length is too small

            0 != (header.length & 3) ||              // packet length is not a multiple of 4

            filesize < offset + header.length) {     // packet would extend beyond the end of the file

                offset++;

                continue;

        }

        // Compute the MD5 Hash of the packet

        MD5Context context;

        context.Update(&header.setid, sizeof(header)-offsetof(PACKET_HEADER, setid));

        // How much more do I need to read to get the whole packet

        u64 current = offset+sizeof(PACKET_HEADER);

        u64 limit = offset+header.length;

        while (current < limit) {

            size_t want = (size_t)min((u64)buffersize, limit-current);

            if (!file->Read(current, buffer, want))

                break;

            context.Update(buffer, want);

            current += want;

        }

        // Did the whole packet get processed

        if (current < limit) {

            offset++;

            continue;

        }

        // Check the calculated packet hash against the value in the header

        MD5Hash hash;

        context.Final(hash);

        if (hash != header.hash) {

            offset++;

            continue;

        }

        // The Set ID is now validated, generate the PAR2Set structure

        *par2set = (PAR2Set*) g_slice_new0(PAR2Set);

        memcpy((*par2set)->set_id, header.setid.hash, sizeof(header.setid.hash));

        if (fileverificationpacket_type == header.type)  {

            //if (LoadVerificationPacket(file, offset, header)) {

                packets++;

            //}

            // Advance to the next packet

            offset += header.length;

        }

    }

    delete [] buffer;

    // We have finished with the file for now

    file->Close();

    // Did we actually find any interesting packets

    if (packets == 0) {

        if ((*par2set)) {

            g_slice_free(PAR2Set, (*par2set));

        }

        delete packet;

        delete file;

        return FALSE;

    }

    delete packet;

    delete file;

#endif

    return TRUE;

}

char *

strip_large_filenames(const char *filename)

{

    /* par2cmdline strips filenames which are larger than 56 characters. 

     * With this function, the same algoritm to strip such filenames can be applied */

    char filename_stripped[1024];

    int len = strlen(filename);

    if (len < 56) {

        return g_strdup(filename);

    }

    memset(filename_stripped, 0, sizeof(filename_stripped));

    strncpy(filename_stripped, filename, 28);

    strcat(filename_stripped, "...");

    strncat(filename_stripped, filename + len - 28, sizeof(filename_stripped) - 28 - 3 - 1);

    return g_strdup(filename_stripped);

}

struct _par2_repair_file {

    char filename[PATH_MAX];

    double verify_progress;

    int num_packets_found;

    int num_blocks_found;

    int num_duplicate_blocks_found;

    int num_blocks_expected;

    gboolean is_missing;

    gboolean is_complete;

    time_t stamp_last_progress_update;

};

static time_t stamp_last_repair_progress_update = 0;

static struct _par2_repair_file *

process_message(NGPlugin *plugin_data, GHashTable *files, struct _par2_repair_file *active_file, const char *line, ngboolean more_par2_sets_remaining_in_collection)

{

    struct _par2_repair_file *file = NULL;

    char *filename = NULL;

    char *filename_stripped = NULL;

    PluginPAR2Priv *priv = plugin_data->priv;

    const char *params[7];

    char progress_str[16];

    char num_packets_found_str[16];

    char num_blocks_found_str[16];

    char num_blocks_expected_str[16];

    // NOTE: par2cmdline can do wrapping of the filename!

    memset(&progress_str, 0, sizeof(progress_str));

    memset(&num_packets_found_str, 0, sizeof(num_packets_found_str));

    memset(&num_blocks_found_str, 0, sizeof(num_blocks_found_str));

    memset(&num_blocks_expected_str, 0, sizeof(num_blocks_expected_str));

    g_return_val_if_fail(priv->collection_name != NULL, NULL);

    g_return_val_if_fail(priv->par2filename != NULL, NULL);

    if (!strncmp(line, "Loading \"", 9)) {                  // Loading "Top 40 dossier _1994 final_.par2".

        char *end = strstr(line + 10, "\"");

        g_return_val_if_fail(end != NULL, NULL);

        filename = g_strndup(line + 9, end - line - 9);

        filename_stripped = strip_large_filenames(filename);

        file = g_hash_table_lookup(files, filename_stripped);

        if (!file) {

            file = g_slice_new0(struct _par2_repair_file);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            g_hash_table_insert(files, g_strdup(filename_stripped), file);

        }

        g_free(filename_stripped);

        g_free(filename);

        snprintf(progress_str, sizeof(progress_str) - 1, "%.2f", file->verify_progress);

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = file->filename;

        params[3] = progress_str;

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_load_progress_update", params);

    } else if (!strncmp(line, "Loading: ", 9)) {            // Loading: 97.9%

        g_return_val_if_fail(active_file != NULL, NULL);

        active_file->verify_progress = g_ascii_strtod(line + 9, NULL);

        file = active_file;

        if (active_file->stamp_last_progress_update != time(NULL)) {

            snprintf(progress_str, sizeof(progress_str) - 1, "%.2f", file->verify_progress);

            params[0] = priv->collection_name;

            params[1] = priv->par2filename;

            params[2] = active_file->filename;

            params[3] = progress_str;

            params[4] = NULL;

            ng_plugin_emit_event(plugin_data, "par2_load_progress_update", params);

            active_file->stamp_last_progress_update = time(NULL);

        }

    } else if (!strncmp(line, "Loaded ", 7)) {              // Loaded 4 new packets

                                                            // Loaded 12 new packets including 12 recovery blocks

        g_return_val_if_fail(active_file != NULL, NULL);

        if (sscanf(line, "Loaded %d new packets including %d recovery blocks", &active_file->num_packets_found, &active_file->num_blocks_found) != 1) {

            // line == 'Loaded x new packets'

            active_file->num_packets_found = atoi(line + 7);

        }

        active_file->verify_progress = 100.0;

        snprintf(num_packets_found_str, sizeof(num_packets_found_str) - 1, "%i", active_file->num_packets_found);

        snprintf(num_blocks_found_str, sizeof(num_blocks_found_str) - 1, "%i", active_file->num_blocks_found);

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = active_file->filename;

        params[3] = num_packets_found_str;

        params[4] = num_blocks_found_str;

        params[5] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_recovery_file_loaded", params);

        /* Maintain a list of PAR2 files which can be removed after the verification/reparation has succeeded */

        priv->par2files = g_list_append(priv->par2files, g_strdup(active_file->filename));

    } else if (!strncmp(line, "No new packets found", 20)) {

        g_return_val_if_fail(active_file != NULL, NULL);

        active_file->num_packets_found = 0;

        active_file->verify_progress = 100.0;

        snprintf(num_packets_found_str, sizeof(num_packets_found_str) - 1, "%i", active_file->num_packets_found);

        snprintf(num_blocks_found_str, sizeof(num_blocks_found_str) - 1, "%i", active_file->num_blocks_found);

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = active_file->filename;

        params[3] = num_packets_found_str;

        params[4] = num_blocks_found_str;

        params[5] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_recovery_file_loaded", params);

        /* Maintain a list of PAR2 files which can be removed after the verification/reparation has succeeded */

        priv->par2files = g_list_append(priv->par2files, g_strdup(active_file->filename));

    } else if (!strncmp(line, "Scanning: \"", 11)) {        // Scanning: "post832.rar": 0.5%

        char *end = strstr(line + 12, "\"");

        g_return_val_if_fail(end != NULL, NULL);

        filename = g_strndup(line + 11, end - line - 11);

        filename_stripped = strip_large_filenames(filename);

        file = g_hash_table_lookup(files, filename_stripped);

        /* On Win32 environments, it can happen that the filename is saved in a different 

         * character encoding than UTF-8. This can confuse the code above, so add a ugly

         * workaround for these kind of situations */

        if (!file) {

            file = g_slice_new0(struct _par2_repair_file);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            g_hash_table_insert(files, g_strdup(filename_stripped), file);

        }

        g_free(filename_stripped);

        g_free(filename);

        g_return_val_if_fail(file != NULL, NULL);

        file->verify_progress = g_ascii_strtod(end + 3, NULL);

        if (file->stamp_last_progress_update != time(NULL)) {

            snprintf(progress_str, sizeof(progress_str) - 1, "%.2f", file->verify_progress);

            params[0] = priv->collection_name;

            params[1] = priv->par2filename;

            params[2] = file->filename;

            params[3] = progress_str;

            params[4] = NULL;

            ng_plugin_emit_event(plugin_data, "par2_load_progress_update", params);

            file->stamp_last_progress_update = time(NULL);

        }

    } else if (!strncmp(line, "Target: \"", 9)) {           // Target: "post832.rar" - found.

        char *end = strstr(line, "\" - ");

        const char *state_str = NULL;

        g_return_val_if_fail(end != NULL, NULL);

        filename = g_strndup(line + 9, end - line - 9);

        filename_stripped = strip_large_filenames(filename);

        file = g_hash_table_lookup(files, filename_stripped);

        // The file can be missing, add a fallback for that situation

        if (!file) {

            file = g_slice_new0(struct _par2_repair_file);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            g_hash_table_insert(files, g_strdup(filename_stripped), file);

        }

        g_free(filename_stripped);

        g_free(filename);

        g_return_val_if_fail(file != NULL, NULL);

        end += 4;

        if (!strncmp(end, "found", 5)) {

            file->is_complete = TRUE;

            state_str = "FOUND";

        } else if (!strncmp(end, "missing", 7)) {

            file->is_missing = TRUE;

            state_str = "MISSING";

        } else if (!strncmp(end, "damaged", 7)) {

            if (strstr(end, "from several target files")) {

                state_str = "NO_NEW_BLOCKS_FOUND";

            } else {

                g_return_val_if_fail (sscanf(end, "damaged. Found %d of %d data blocks.", &file->num_blocks_found, &file->num_blocks_expected) == 2, NULL);

                state_str = "DAMAGED";

            }

        } else {

            g_print(__FILE__ ":%i reason = %s\n", __LINE__, end);

            state_str = "MISSING";

        }

        snprintf(num_blocks_found_str, sizeof(num_blocks_found_str) - 1, "%i", file->num_blocks_found);

        snprintf(num_blocks_expected_str, sizeof(num_blocks_expected_str) - 1, "%i", file->num_blocks_expected);

        params[0] = priv->collection_name,

        params[1] = priv->par2filename;

        params[2] = file->filename;

        params[3] = state_str;

        params[4] = num_blocks_found_str;

        params[5] = num_blocks_expected_str;

        params[6] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_file_loaded", params);

    } else if (!strncmp(line, "File: \"", 7)) {             // File: "1994_026_CDS__TW_ Prodigy - No Good _Start The Dance_ _Original Mix_.mp3" - found 12 of 13 data blocks from "1994_026[CDS][TW] Prodigy - No Good (Start The Dance) (Original Mix).mp3".

        char *end = strstr(line, "\" - ");

        const char *state_str;

        g_return_val_if_fail(end != NULL, NULL);

        filename = g_strndup(line + 7, end - line - 7);

        filename_stripped = strip_large_filenames(filename);

        file = g_hash_table_lookup(files, filename_stripped);

        // The file can be missing, add a fallback for that situation

        if (!file) {

            file = g_slice_new0(struct _par2_repair_file);

            strncpy(file->filename, filename, sizeof(file->filename) - 1);

            g_hash_table_insert(files, g_strdup(filename_stripped), file);

        }

        g_free(filename);

        g_return_val_if_fail(file != NULL, NULL);

        end += 4;

        if (strncmp(end, "no data found.", 14) &&

            strncmp(end, "is a match for", 14) &&

            !strstr(end, "duplicate data blocks.") &&

            !strstr(end, "data blocks from several target files")) {

            g_return_val_if_fail (sscanf(end, "found %d of %d data blocks", &file->num_blocks_found, &file->num_blocks_expected) == 2, NULL);

            if (file->num_blocks_found == file->num_blocks_expected) {

                state_str = "FOUND";

            } else {

                state_str = "DAMAGED";

            }

        } else if (!strncmp(end, "no data found.", 14) ||

                    strstr(end, "data blocks from several target files") ||

                    strstr(end, "duplicate data blocks.")) {

            state_str = "NO_NEW_BLOCKS_FOUND";

        } else if (!strncmp(end, "is a match for", 14)) {

            file->num_blocks_found = file->num_blocks_expected = 1;

            file->verify_progress = 100.0;

            state_str = "FOUND";

        } else {

g_print(__FILE__ ":%i unknown state for PAR2 message: %s (end = %s)\n", __LINE__, line, end);

            state_str = "MISSING";

        }

        g_free(filename_stripped);

        snprintf(num_blocks_found_str, sizeof(num_blocks_found_str) - 1, "%i", file->num_blocks_found);

        snprintf(num_blocks_expected_str, sizeof(num_blocks_expected_str) - 1, "%i", file->num_blocks_expected);

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = file->filename;

        params[3] = state_str;

        params[4] = num_blocks_found_str;

        params[5] = num_blocks_expected_str;

        params[6] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_file_loaded", params);

    } else if (!strncmp(line, "You need ", 9)) {         // You need 3000 more recovery blocks to be able to repair

        char num_blocks_required_str[16];

        int num_blocks_required = atoi(line + 9);

        priv->repair_succeeded = FALSE;

        priv->command_completed_normally = TRUE;

        memset(&num_blocks_required_str, 0, sizeof(num_blocks_required_str));

        snprintf(num_blocks_required_str, sizeof(num_blocks_required_str) - 1, "%i", num_blocks_required);

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = "";

        params[3] = num_blocks_required_str;

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

    } else if (!strncmp(line, "Repairing:", 10)) {

        double progress = g_ascii_strtod(line + 11, NULL);

        if (stamp_last_repair_progress_update != time(NULL)) {

            snprintf(progress_str, sizeof(progress_str) - 1, "%.2f", progress);

            params[0] = priv->collection_name;

            params[1] = priv->par2filename;

            params[2] = progress_str;

            params[3] = NULL;

            ng_plugin_emit_event(plugin_data, "par2_repair_progress_update", params);

            stamp_last_repair_progress_update = time(NULL);

        }

    } else if (!strncmp(line, "Repair complete.", 16)) {

        priv->repair_succeeded = TRUE;

        priv->command_completed_normally = TRUE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        if (more_par2_sets_remaining_in_collection) {

            params[2] = "TRUE";

        } else {

            params[2] = "FALSE";

        }

        params[3] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_success", params);

    } else if (!strncmp(line, "All files are correct, repair is not required.", 46)) {

        priv->repair_succeeded = TRUE;

        priv->command_completed_normally = TRUE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        if (more_par2_sets_remaining_in_collection) {

            params[2] = "TRUE";

        } else {

            params[2] = "FALSE";

        }

        params[3] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_no_repair_required", params);

    } else if (!strncmp(line, "Could not read", 14)) {

        priv->repair_succeeded = FALSE;

        priv->command_completed_normally = TRUE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = line;

        params[3] = "-1";

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

    } else if (!strncmp(line, "The recovery file does not exist", 32)) {

        priv->repair_succeeded = FALSE;

        priv->command_completed_normally = TRUE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = "Bug in par2cmdline was triggered";

        params[3] = "-1";

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

    } else if (strstr(line, "cannot be renamed to")) {

        priv->repair_succeeded = FALSE;

        priv->command_completed_normally = TRUE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = "Error: par2cmdline wasn't able to rename a file";

        params[3] = "-1";

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

    } else if (!strncmp(line, "pid", 3)) {

        priv->pid = atoi(line + 3);

    } else {

// g_print("line = %s - active_file = %#x\n", line, active_file);

    }

    /* 

    Loading "Top 40 dossier _1994 final_.par2".

    Loaded 5 new packets including 1 recovery blocks

    Repair is required.

    Repair is not possible.

    You need 3000 more recovery blocks to be able to repair.

    Loading: 99.9%

    Loaded 712 new packets

    No new packets found

    There are 355 recoverable files and 0 other files.

    The block size used was 1223232 bytes.

    There are a total of 3000 data blocks.

    The total size of the data files is 3453446279 bytes.

    Verifying source files:

    Scanning: "post832.rar": 0.5%

    Target: "post832.rar" - found.

    Target: "1994_001[CD][ME] Marco Borsato - Dromen Zijn Bedrog.mp3" - missing.

    Scanning extra files:

    File: "1994_026_CDS__TW_ Prodigy - No Good _Start The Dance_ _Original Mix_.mp3" - found 12 of 13 data blocks from "1994_026[CDS][TW] Prodigy - No Good (Start The Dance) (Original Mix).mp3".

    File: "vrllvhnbnz.part032.rar" - no data found.

    All files are correct, repair is not required.

    */

    return file;

}

static void

par2_repair_file_free(gpointer data)

{

    g_slice_free(struct _par2_repair_file, data);

}

static gboolean

verify_par2_extension(const char *filename)

{

    char *ptr = g_strrstr(filename, ".");

    if (!ptr) {

        /* File doesn't have an extension */

        return FALSE;

    }

    /* Does the filename have an '.par' or '.par2' extension? */

    if (!g_ascii_strncasecmp(ptr, ".par2", 5) || !g_strncasecmp(ptr, ".par", 4)) {

        return TRUE;

    }

    /* Test for files with the extension '.par.1' or '.par2.1' */

    ptr = g_strrstr(filename, ".");

    if (!ptr) {

        return FALSE;

    }

    if (!g_ascii_strncasecmp(ptr, ".par2.", 6) || !g_strncasecmp(ptr, ".par.", 5)) {

        return TRUE;

    }

    return FALSE;

}

static gboolean

par2_start_par2cmdline(NGPlugin *plugin_data, const char *collection_name, const char *directory, const char *par2filename, GHashTable *files, gboolean try_to_disable_concurrent_verify, gboolean *invalid_option_detected, ngboolean more_par2_sets_remaining_in_collection)

{

    FILE *fp;

    char *cmd;

    char buf[1024];

    int i;

    int c;

    struct _par2_repair_file *active_file = NULL;

    const char *disable_concurrent_verify_option;

    gboolean first_line = TRUE;

#ifndef WIN32

    gboolean second_line = FALSE;

#endif

    PluginPAR2Priv *priv;

    static GStaticMutex mutex = G_STATIC_MUTEX_INIT;

    NGConfigOpts opts;

    GList *list;

#ifdef WIN32

    char *filename;

    char *dirname = g_locale_from_utf8(directory, -1, NULL, NULL, NULL);

    char *bindir = g_path_get_dirname(__argv[0]);

    char *cmd2;

    g_return_val_if_fail(dirname != NULL, FALSE);

#endif

    priv = plugin_data->priv;

    if (priv->abort_flag) {

        const char *params[5];

        priv->repair_succeeded = FALSE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = "Error: par2cmdline aborted unexpectedly";

        params[3] = "-1";

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

        return FALSE;

    }

    g_static_mutex_lock(&mutex);

    if (try_to_disable_concurrent_verify) {

        disable_concurrent_verify_option = "-t0";

    } else {

        disable_concurrent_verify_option = "";

    }

#ifdef WIN32

    filename = g_locale_from_utf8(par2filename, -1, NULL, NULL, NULL);

    g_return_val_if_fail(filename != NULL, FALSE);

    cmd = g_strdup_printf("\"%s\\par2.exe\" r %s \"%s\" \"%s"G_DIR_SEPARATOR_S"*.*\" 2>&1", bindir, disable_concurrent_verify_option, filename, dirname);

    g_free(filename);

    g_free(dirname);

    g_free(bindir);

#else

    cmd = g_strdup_printf("echo \"pid$$\"; par2 r %s \"%s\" \"%s"G_DIR_SEPARATOR_S"\"*.* 2>&1", disable_concurrent_verify_option, par2filename, directory);

#endif

    fp = popen(cmd, "r");

#ifdef WIN32

    cmd2 = g_locale_to_utf8(cmd, -1, NULL, NULL, NULL);

    g_return_val_if_fail(cmd2 != NULL, FALSE);

    ng_plugin_emit_log_msg(plugin_data, 1, _("Launched the command: %s\n"), cmd2);

    g_free(cmd2);

#else

    ng_plugin_emit_log_msg(plugin_data, 1, _("Launched the command: %s\n"), cmd);

#endif

    if (!fp) {

        ng_plugin_emit_log_msg(plugin_data, 1, _("Error while launching from the par2cmdline process\n%s\n%s"), cmd, strerror(errno));

        g_free(cmd);

        g_static_mutex_unlock(&mutex);

        return FALSE;

    }

    g_free(cmd);

    priv->collection_name = collection_name;

    priv->par2filename = par2filename;

    priv->par2files = NULL;

    priv->pid = -1;

    // Assume the repair was succesfull unless it explicit failed

    priv->repair_succeeded = TRUE;

    // Detect abnormal aborts of par2cmdline

    priv->command_completed_normally = FALSE;

    memset(buf, 0, sizeof(buf));

    i = 0;

    while ((c = fgetc(fp)) != EOF) {

        if (priv->abort_flag) {

            break;

        }

        if (c == '\r' || (c == '\n' && i > 0)) {

            if (first_line) {

                first_line = FALSE;

#ifndef WIN32

                second_line = TRUE;

            } else if (second_line) {

                second_line = FALSE;

#endif

                /* Check if the given options were valid */

                if (!strncmp(buf, "Invalid option specified", 24)) {

                    if (invalid_option_detected) {

                        *invalid_option_detected = TRUE;

                    }

                    pclose(fp);

                    priv->collection_name = NULL;

                    priv->par2filename = NULL;

                    g_static_mutex_unlock(&mutex);

                    return FALSE;

                }

            }

            /* Translate the line to UTF-8 if it isn't already */

            if (!g_utf8_validate(buf, -1, NULL)) {

                /* Filename is probably in Windows-1252 charset */

                char *tmp = g_convert(buf, -1, "utf-8", "windows-1252", NULL, NULL, NULL);

                if (tmp) {

                    memset(buf, 0, sizeof(buf));

                    strncpy(buf, tmp, sizeof(buf) - 1);

                    g_free(tmp);

                }

            }

            i = 0;

            active_file = process_message(plugin_data, files, active_file, buf, more_par2_sets_remaining_in_collection);

// imported_funcs.emit_repair_message(buf);

            memset(buf, 0, sizeof(buf));

        } else if (c == '\n') {

            // Ignore

        } else {

            buf[i] = (char) c;

            i++;

        }

    }

    pclose(fp);

    if (!priv->command_completed_normally) {

        /* Abnormal abort of par2cmdline detected, send a notification to the frontend */

        const char *params[5];

        priv->repair_succeeded = FALSE;

        params[0] = priv->collection_name;

        params[1] = priv->par2filename;

        params[2] = "Error: par2cmdline aborted unexpectedly";

        params[3] = "-1";

        params[4] = NULL;

        ng_plugin_emit_event(plugin_data, "par2_repair_failure", params);

    }

    /* Remove the PAR2 files after a successfull verification/recovery if it's enabled */

    opts = plugin_data->core_funcs.config_get_opts();

    list = priv->par2files;

    while (list) {

        char *filename = list->data;

        /* Make sure that the file really is a PAR/PAR2 file. Due to a bug in par2cmdline, regular files might get marked as PAR2 recovery files */

        if (priv->repair_succeeded && !more_par2_sets_remaining_in_collection && opts.auto_remove_files_after_repair) {

            if (verify_par2_extension(filename)) {

                char *path = g_build_filename(directory, filename, NULL);

                ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, _("Now automatically removing file '%s'"), path);

                g_unlink(path);

                g_free(path);

            } else {

                ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, _("File '%s' was marked as a PAR2 recovery file by par2cmdline while it actually isn't. Ignoring"), filename);

            }

        }

        g_free(filename);

        list = g_list_next(list);

    }

    g_list_free(priv->par2files);

    priv->collection_name = NULL;

    priv->par2filename = NULL;

    g_static_mutex_unlock(&mutex);

    return priv->repair_succeeded;

}

static gboolean

nntpgrab_plugin_par2_repair_files(NGPlugin *plugin_data, const char *collection_name, const char *directory, const char *par2filename, ngboolean more_par2_sets_remaining_in_collection)

{

    char *path;

    GHashTable *files = NULL;

    GDir *dir;

    GError *error = NULL;

    const char *filename;

    char *filename_stripped;

    gboolean invalid_option_detected = FALSE;

    gboolean repair_succeeded;

    path = g_build_filename(directory, par2filename, NULL);

    g_return_val_if_fail(g_file_test(path, G_FILE_TEST_EXISTS), FALSE);

    // Build a list of all the files which are in the directory

    dir = g_dir_open(directory, 0, &error);

    if (!dir) {

        ng_plugin_emit_log_msg(plugin_data, 1, _("PAR2 repair can't be started because the directory with the\ndownloaded files can't be opened: %s"), error->message);

        g_error_free(error);

        return FALSE;

    }

    files = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, par2_repair_file_free);

    while ((filename = g_dir_read_name(dir))) {

        struct _par2_repair_file *file = g_slice_new0(struct _par2_repair_file);

        strncpy(file->filename, filename, sizeof(file->filename) - 1);

        filename_stripped = strip_large_filenames(filename);

        g_hash_table_insert(files, filename_stripped, file);

        /* No need to g_free(filename_stripped) here as that will be done by the GHashTable itself */

    }

    g_dir_close(dir);

    ng_plugin_emit_log_msg(plugin_data, 1, "%s", _("Now starting PAR2 repair"));

    repair_succeeded = par2_start_par2cmdline(plugin_data, collection_name, directory, path, files, TRUE, &invalid_option_detected, more_par2_sets_remaining_in_collection);

    if (!repair_succeeded && invalid_option_detected) {

        /* The regular par2cmdline was detected, try again with the -t0 option */

        repair_succeeded = par2_start_par2cmdline(plugin_data, collection_name, directory, path, files, FALSE, NULL, more_par2_sets_remaining_in_collection);

    }

    if (repair_succeeded) {

        ng_plugin_emit_log_msg(plugin_data, 1, "%s", _("PAR2 repair finished successfully"));

    } else {

        ng_plugin_emit_log_msg(plugin_data, 1, "%s", _("PAR2 repair failed"));

    }

    g_free(path);

    g_hash_table_destroy(files);

    return repair_succeeded;

}

typedef struct _par2_file_seen {

    char filename_without_extension[256];

    char filename[256];

} PAR2FileSeen;

/* APARTE THREAD ! */

struct _par2_thread_data {

    NGPlugin *plugin_data;

    char collection_name[256];

};

static void

par2_thread_func(gpointer data, gpointer user_data)

{

    struct _par2_thread_data *thread_data = (struct _par2_thread_data *) data;

    NGConfigOpts opts;

    char *directory;

    GList *list;

    GList *list2;

    GList *files = NULL;

    GDir *dir;

    GError *err = NULL;

    const char *filename;

    /* Do we need to repair? */

    opts = thread_data->plugin_data->core_funcs.config_get_opts();

    if (!opts.enable_par2_repair || thread_data->plugin_data->core_funcs.schedular_get_state() != SCHEDULAR_STATE_RUNNING || ((PluginPAR2Priv*)thread_data->plugin_data->priv)->abort_flag == TRUE) {

        g_slice_free(struct _par2_thread_data, thread_data);

        return;

    }

    /* Find out all the unique PAR2 sets in this collection */

    directory = g_build_filename(opts.download_directory, thread_data->collection_name, NULL);

    dir = g_dir_open(directory, 0, &err);

    if (!dir) {

        ng_plugin_emit_log_msg(thread_data->plugin_data, NG_LOG_LEVEL_WARNING, _("Unable to open directory '%s': %s"), directory, err->message);

        g_error_free(err);

        g_free(directory);

        g_slice_free(struct _par2_thread_data, thread_data);

        return;

    }

    while ((filename = g_dir_read_name(dir))) {

        char *extension;

        char filename_without_extension[256];

        gboolean filename_seen = FALSE;

        char *ptr;

        extension = g_strrstr(filename, ".");

        if (!extension) {

            /* File without extension, ignore */

            continue;

        }

        if (g_ascii_strncasecmp(extension, ".par", 4) != 0) {

            /* Different extension, ignore */

            continue;

        }

        memset(filename_without_extension, 0, sizeof(filename_without_extension));

        strncpy(filename_without_extension, filename, extension - filename);

        // Strip any .volXX+YY part

        ptr = g_strrstr(filename_without_extension, ".");

        if (ptr && !g_strncasecmp(ptr, ".vol", 4)) {

            *ptr = '\0';

        }

        list2 = files;

        while (list2) {

            PAR2FileSeen *file = (PAR2FileSeen*) list2->data;

            if (!strcmp(filename_without_extension, file->filename_without_extension)) {

                // We've already seen this PAR2 set, ignore

                filename_seen = TRUE;

                break;

            }

            list2 = g_list_next(list2);

        }

        if (!filename_seen) {

            PAR2FileSeen *file_seen = g_slice_new0(PAR2FileSeen);

            strncpy(file_seen->filename_without_extension, filename_without_extension, sizeof(file_seen->filename_without_extension) - 1);

            strncpy(file_seen->filename, filename, sizeof(file_seen->filename) - 1);

            files = g_list_append(files, file_seen);

        }

    }

    g_dir_close(dir);

    if (!files) {

        ng_plugin_emit_log_msg(thread_data->plugin_data, NG_LOG_LEVEL_DEBUG, _("No .par2 files found for collection '%s'"), thread_data->collection_name);

    }

    /* Perform the repair */

    list = files;

    while (list) {

        PAR2FileSeen *file_seen = (PAR2FileSeen*) list->data;

        nntpgrab_plugin_par2_repair_files(thread_data->plugin_data, thread_data->collection_name, directory, file_seen->filename, (list->next != NULL));

        g_slice_free(PAR2FileSeen, file_seen);

        list = g_list_next(list);

    }

    g_list_free(files);

    g_free(directory);

    g_slice_free(struct _par2_thread_data, thread_data);

}

static void

on_collection_downloaded(NGPlugin *plugin_data, const char *collection_name, gpointer data)

{

    struct _par2_thread_data *thread_data = g_slice_new0(struct _par2_thread_data);

    thread_data->plugin_data = plugin_data;

    strncpy(thread_data->collection_name, collection_name, sizeof(thread_data->collection_name) - 1);

    g_thread_pool_push(((PluginPAR2Priv *) plugin_data->priv)->thread_pool, thread_data, NULL);

}