NNTPGrab

/* 

    Copyright (C) 2005-2010  Erik van Pienbroek

    This program is free software; you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by

    the Free Software Foundation; either version 2 of the License, or

    (at your option) any later version.

    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License

    along with this program; if not, write to the Free Software

    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

*/

#include 

#include 

#include 

#include 

#include 

#include 

#include 

#include 

#include "nntpgrab_plugin.h"

#include "yydecode.h"

#include "crc32.h"

#include "marshalers.h"

#include "config.h"

// yydecode.c

int decode(const char *inname, const char *forced_outname, struct decoded_file **decoded_list, int *saved_errno, char **errmsg);

static NNTPGrabDecoderRes

nntpgrab_plugin_decoder_decode_file(NGPlugin *plugin_data, const char *collection_name, NNTPFile *file, const char *temp_directory, const char *target_directory, int *saved_errno, char **errmsg);

void

nntpgrab_plugin_initialize(NGPlugin *plugin_data)

{

    ng_plugin_set_name(plugin_data, "Decoder");

    ng_plugin_set_version(plugin_data, PACKAGE_VERSION);

    ng_plugin_set_author(plugin_data, "Erik van Pienbroek");

    ng_plugin_set_url(plugin_data, "https://www.nntpgrab.nl");

    ng_plugin_set_description(plugin_data, "A plugin which is responsible for decoding yEnc and UU-encoded files");

    ng_plugin_register_function(plugin_data,

                                "decode_file",

                                NG_PLUGIN_FUNCTION(nntpgrab_plugin_decoder_decode_file),

                                ng_plugin_marshal_INT__STRING_POINTER_STRING_STRING_POINTER_POINTER,

                                G_TYPE_INT,

                                6, G_TYPE_STRING, G_TYPE_POINTER, G_TYPE_STRING,

                                   G_TYPE_STRING, G_TYPE_POINTER, G_TYPE_POINTER);

}

ngboolean

nntpgrab_plugin_load(NGPlugin *plugin_data, char **errmsg)

{

    return TRUE;

}

ngboolean

nntpgrab_plugin_can_unload(NGPlugin *plugin_data, char **reason)

{

    return TRUE;

}

void

nntpgrab_plugin_unload(NGPlugin *plugin_data)

{

}

void

nntpgrab_plugin_destroy(NGPlugin *plugin_data)

{

}

int

nntpgrab_plugin_get_version(void)

{

    return NNTPGRAB_PLUGIN_API_VERSION;

}

/* MD5 code copied from plugins/jsonrpc/mongoose.c */

typedef struct MD5Context {

        guint32         buf[4];

        guint32         bits[2];

        unsigned char   in[64];

} MD5_CTX;

#if __BYTE_ORDER == 1234

#define byteReverse(buf, len)        /* Nothing */

#else

/* 

 * Note: this code is harmless on little-endian machines.

 */

static void

byteReverse(unsigned char *buf, unsigned longs)

{

        guint32 t;

        do {

                t = (guint32) ((unsigned) buf[3] << 8 | buf[2]) << 16 |

                        ((unsigned) buf[1] << 8 | buf[0]);

                *(guint32 *) buf = t;

                buf += 4;

        } while (--longs);

}

#endif /* __BYTE_ORDER */

/* The four core functions - F1 is optimized somewhat */

/* #define F1(x, y, z) (x & y | ~x & z) */

#define F1(x, y, z) (z ^ (x & (y ^ z)))

#define F2(x, y, z) F1(z, x, y)

#define F3(x, y, z) (x ^ y ^ z)

#define F4(x, y, z) (y ^ (x | ~z))

/* This is the central step in the MD5 algorithm. */

#define MD5STEP(f, w, x, y, z, data, s) \

( w += f(x, y, z) + data,  w = w<>(32-s),  w += x )

/* 

 * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious

 * initialization constants.

 */

static void

MD5Init(MD5_CTX *ctx)

{

        ctx->buf[0] = 0x67452301;

        ctx->buf[1] = 0xefcdab89;

        ctx->buf[2] = 0x98badcfe;

        ctx->buf[3] = 0x10325476;

        ctx->bits[0] = 0;

        ctx->bits[1] = 0;

        memset(ctx->in, 0, sizeof(ctx->in));

}

/* 

 * The core of the MD5 algorithm, this alters an existing MD5 hash to

 * reflect the addition of 16 longwords of new data.  MD5Update blocks

 * the data and converts bytes into longwords for this routine.

 */

static void

MD5Transform(guint32 buf[4], guint32 const in[16])

{

        register guint32 a, b, c, d;

        a = buf[0];

        b = buf[1];

        c = buf[2];

        d = buf[3];

        MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);

        MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);

        MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);

        MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);

        MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);

        MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);

        MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);

        MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);

        MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);

        MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);

        MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);

        MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);

        MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);

        MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);

        MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);

        MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);

        MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);

        MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);

        MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);

        MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);

        MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);

        MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);

        MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);

        MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);

        MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);

        MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);

        MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);

        MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);

        MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);

        MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);

        MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);

        MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);

        MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);

        MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);

        MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);

        MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);

        MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);

        MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);

        MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);

        MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);

        MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);

        MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);

        MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);

        MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);

        MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);

        MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);

        MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);

        MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);

        MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);

        MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);

        MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);

        MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);

        MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);

        MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);

        MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);

        MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);

        MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);

        MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);

        MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);

        MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);

        MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);

        MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);

        MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);

        MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);

        buf[0] += a;

        buf[1] += b;

        buf[2] += c;

        buf[3] += d;

}

/* 

 * Update context to reflect the concatenation of another buffer full

 * of bytes.

 */

static void

MD5Update(MD5_CTX *ctx, unsigned char const *buf, unsigned len)

{

        guint32 t;

        if (len == 0) {

            return;

        }

        /* Update bitcount */

        t = ctx->bits[0];

        if ((ctx->bits[0] = t + ((guint32) len << 3)) < t)

                ctx->bits[1]++;                /* Carry from low to high */

        ctx->bits[1] += len >> 29;

        t = (t >> 3) & 0x3f;        /* Bytes already in shsInfo->data */

        /* Handle any leading odd-sized chunks */

        if (t) {

                unsigned char *p = (unsigned char *) ctx->in + t;

                t = 64 - t;

                if (len < t) {

                        memcpy(p, buf, len);

                        return;

                }

                memcpy(p, buf, t);

                byteReverse(ctx->in, 16);

                MD5Transform(ctx->buf, (guint32 *) ctx->in);

                buf += t;

                len -= t;

        }

        /* Process data in 64-byte chunks */

        while (len >= 64) {

                memcpy(ctx->in, buf, 64);

                byteReverse(ctx->in, 16);

                MD5Transform(ctx->buf, (guint32 *) ctx->in);

                buf += 64;

                len -= 64;

        }

        /* Handle any remaining bytes of data. */

        memcpy(ctx->in, buf, len);

}

/* 

 * Final wrapup - pad to 64-byte boundary with the bit pattern

 * 1 0* (64-bit count of bits processed, MSB-first)

 */

static void

MD5Final(unsigned char digest[16], MD5_CTX *ctx)

{

        unsigned count;

        unsigned char *p;

        /* Compute number of bytes mod 64 */

        count = (ctx->bits[0] >> 3) & 0x3F;

        /* Set the first char of padding to 0x80. This is safe since there is 

           always at least one byte free */

        p = ctx->in + count;

        *p++ = 0x80;

        /* Bytes of padding needed to make 64 bytes */

        count = 64 - 1 - count;

        /* Pad out to 56 mod 64 */

        if (count < 8) {

                /* Two lots of padding: Pad the first block to 64 bytes */

                memset(p, 0, count);

                byteReverse(ctx->in, 16);

                MD5Transform(ctx->buf, (guint32 *) ctx->in);

                /* Now fill the next block with 56 bytes */

                memset(ctx->in, 0, 56);

        } else {

                /* Pad block to 56 bytes */

                memset(p, 0, count - 8);

        }

        byteReverse(ctx->in, 14);

        /* Append length in bits and transform */

        ((guint32 *) ctx->in)[14] = ctx->bits[0];

        ((guint32 *) ctx->in)[15] = ctx->bits[1];

        MD5Transform(ctx->buf, (guint32 *) ctx->in);

        byteReverse((unsigned char *) ctx->buf, 4);

        memcpy(digest, ctx->buf, 16);

        memset((char *) ctx, 0, sizeof(ctx));        /* In case it's sensitive */

}

static void

strip_newline(char *line)

{

    if (line[strlen(line) - 1]  == '\n') {

        line[strlen(line) - 1] = '\0';

    }

    if (line[strlen(line) - 1]  == '\r') {

        line[strlen(line) - 1] = '\0';

    }

}

static gboolean

parse_ybegin_header(const char *line, int *part, int *num_lines, guint64 *size, char *name, int name_length)

{

    int i;

    int len;

    /* line must be in one of the following notations: */

    /* =ybegin line=128 size=100000 name=abc.rar */

    /* =ybegin part=1 line=128 size=50000000 name=abc.rar */

    g_return_val_if_fail(line != NULL, FALSE);

    g_return_val_if_fail(part != NULL, FALSE);

    g_return_val_if_fail(line != NULL, FALSE);

    g_return_val_if_fail(size != NULL, FALSE);

    g_return_val_if_fail(name != NULL, FALSE);

    g_return_val_if_fail(name_length > 0, FALSE);

    *part = -1;

    *num_lines = -1;

    *size = -1;

    memset(name, 0, name_length);

    len = strlen(line);

    for (i = 0; i < len; i++) {

        if (!strncmp(line + i, "part=", 5)) {

            *part = atoi(line + i + 5);

        } else if (!strncmp(line + i, "line=", 5)) {

            *num_lines = atoi(line + i + 5);

        } else if (!strncmp(line + i, "size=", 5)) {

            *size = g_ascii_strtoull (line + i + 5, NULL, 10);

        } else if (!strncmp(line + i, "name=", 5)) {

            strcpy(name, line + i + 5);

            strip_newline(name);

        }

    }

#if 0 

g_print("part = %i\n", *part);

g_print("num_lines = %i\n", *num_lines);

g_print("size = %li\n", *size);

g_print("filename = %s\n\n", name);

#endif

    if (*num_lines > 0 && *size > 0 && strlen(name) > 0) {  /* NOTE: The part field is optional */

        return TRUE;

    } else {

        return FALSE;

    }

}

static gboolean

parse_ypart_header(const char *line, guint64 *begin, guint64 *end)

{

    int i;

    int len;

    /* line must be in the following notation: */

    /* =ypart begin=1 end=640000 */

    g_return_val_if_fail(line != NULL, FALSE);

    g_return_val_if_fail(begin != NULL, FALSE);

    g_return_val_if_fail(end != NULL, FALSE);

    *begin = -1;

    *end = -1;

    len = strlen(line);

    for (i = 0; i < len; i++) {

        if (!strncmp(line + i, "begin=", 6)) {

            *begin = g_ascii_strtoull(line + i + 6, NULL, 10);

        } else if (!strncmp(line + i, "end=", 4)) {

            *end = g_ascii_strtoull(line + i + 4, NULL, 10);

        }

    }

#if 0 

g_print("begin = %li\n", *begin);

g_print("end = %li\n", *end);

#endif

    if (*begin >= 0 && *end > 0) {

        return TRUE;

    } else {

        return FALSE;

    }

}

static gboolean

try_to_use_new_decoder(NGPlugin *plugin_data, const char *filename_in, const char *target_directory, NNTPGrabDecoderRes *decoder_result, char **errmsg, FILE **prev_file_out_fp, char *prev_filename_out, int prev_filename_out_length, char *prev_real_filename_out, int prev_real_filename_out_length, MD5_CTX *md5_ctx)

{

    int part = -1;

    int num_lines = -1;

    guint64 size = -1;

    char filename_out[1024];

    guint64 begin = -1;

    guint64 end = -1;

    FILE *fp;

    gboolean escape_char_detected = FALSE;

    char path[4096];

    char buf[1024];

    struct crc32_ctx crc32_context;

    g_return_val_if_fail(filename_in != NULL, FALSE);

    g_return_val_if_fail(target_directory != NULL, FALSE);

    g_return_val_if_fail(decoder_result != NULL, FALSE);

    /* NOTE: errmsg MIGHT be NULL */

    g_return_val_if_fail(prev_file_out_fp != NULL, FALSE);

    g_return_val_if_fail(prev_filename_out != NULL, FALSE);

    g_return_val_if_fail(prev_filename_out_length > 0, FALSE);

    *decoder_result = DECODER_RESULT_COMPLETE;

    /* Try to read the first line from the given filename to detected if this is an yEnc encoded file */

    fp = g_fopen(filename_in, "rb");

    if (!fp) {

        *decoder_result = DECODER_RESULT_ERROR;

        if (errmsg) {

            *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to open file '%s': %s"), __FILE__, __LINE__, filename_in, strerror(errno));

        }

        return TRUE;

    }

    /* Try to read the header from this file */

    do {

        if (!fgets(buf, sizeof(buf) - 1, fp)) {

            if (feof(fp)) {

                /* The file doesn't contain any data at all */

                *decoder_result = DECODER_RESULT_INCOMPLETE;

                ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, __FILE__ ":%i File %s is empty\n", __LINE__, filename_in);

            } else {

                *decoder_result = DECODER_RESULT_ERROR;

                if (errmsg) {

                    *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to read the first line from file '%s': %s"), __FILE__, __LINE__, filename_in, strerror(errno));

                }

            }

            fclose(fp);

            return TRUE;

        }

        strip_newline(buf);

        if (strlen(buf) == 0) {

            continue;

        }

        if (strncmp(buf, "=ybegin", 7) != 0) {

            /* Other encoding found. Process this with a different decoder */

            fclose(fp);

            return FALSE;

        }

        break;

    } while (TRUE);

    /* yEnc encoded file found! */

    /* The header is now in buf. Try to extract information from this header */

    memset(filename_out, 0, sizeof(filename_out));

    if (!parse_ybegin_header(buf, &part, &num_lines, &size, filename_out, sizeof(filename_out) - 1)) {

        /* Header incomplete. Ignore this file */

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, __FILE__ ":%i yBegin-header incomplete, buf = %s\n", __LINE__, buf);

        *decoder_result = DECODER_RESULT_INCOMPLETE;

        fclose(fp);

        return TRUE;

    }

    /* Translate the filename to UTF-8 if it isn't already */

    if (!g_utf8_validate(filename_out, -1, NULL)) {

        /* Filename is probably in Windows-1252 charset */

        char *tmp = g_convert(filename_out, -1, "utf-8", "windows-1252", NULL, NULL, NULL);

        if (!tmp) {

            *decoder_result = DECODER_RESULT_ERROR;

            if (errmsg) {

                *errmsg = g_strdup_printf(_("%s:%i The filename '%s' is using an unknown character set. Decoding cannot continue"), __FILE__, __LINE__, filename_out);

            }

            fclose(fp);

            return TRUE;

        }

        memset(filename_out, 0, sizeof(filename_out));

        strncpy(filename_out, tmp, sizeof(filename_out) - 1);

        g_free(tmp);

    }

    /* If this is a multipart file, another header is sent */

    if (part != -1) {

        if (!fgets(buf, sizeof(buf) - 1, fp)) {

            *decoder_result = DECODER_RESULT_ERROR;

            if (errmsg) {

                *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to read the first line from file '%s': %s"), __FILE__, __LINE__, filename_in, strerror(errno));

            }

            fclose(fp);

            return TRUE;

        }

        if (!parse_ypart_header(buf, &begin, &end)) {

            /* Header incomplete. Ignore this file */

            ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, __FILE__ ":%i Part-header incomplete, buf = %s\n", __LINE__, buf);

            *decoder_result = DECODER_RESULT_INCOMPLETE;

            fclose(fp);

            return TRUE;

        }

    }

    /* Open the output file (if it wasn't already open) */

    memset(path, 0, sizeof(path));

    g_snprintf(path, sizeof(path) - 1, "%s%s%s", target_directory, G_DIR_SEPARATOR_S, filename_out);

    if (strcmp(filename_out, prev_filename_out) != 0) {

        char path_orig[4096];

        int suffix = 1;

        char *filename_tmp;

        /* Output file wasn't open yet. Open it now */

        if (*prev_file_out_fp != NULL) {

            if (fclose(*prev_file_out_fp) != 0) {

                *decoder_result = DECODER_RESULT_ERROR;

                if (errmsg) {

                    *errmsg = g_strdup_printf(_("%s:%i Unable to close file '%s': %s"), __FILE__, __LINE__, prev_filename_out, strerror(errno));

                }

                *prev_file_out_fp = NULL;

                return TRUE;

            }

        }

        /* If there's already a file at 'path', append a suffix to it and try again */

        strcpy(path_orig, path);

        while (g_file_test(path, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR)) {

            g_snprintf(path, sizeof(path) - 1, "%s.%i", path_orig, suffix);

            suffix++;

        }

        *prev_file_out_fp = g_fopen(path, "wb");

        if (!(*prev_file_out_fp)) {

            *decoder_result = DECODER_RESULT_ERROR;

            if (errmsg) {

                *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to open file for writing '%s': %s"), __FILE__, __LINE__, path, strerror(errno));

            }

            fclose(fp);

            return TRUE;

        }

        strncpy(prev_filename_out, filename_out, prev_filename_out_length);

        filename_tmp = g_path_get_basename(path);

        strncpy(prev_real_filename_out, filename_tmp, prev_real_filename_out_length);

        g_free(filename_tmp);

    }

    g_return_val_if_fail(*prev_file_out_fp != NULL, TRUE);

    /* Seek to the right position in the output file if we're decoding a multi-part file */

    if (begin > 0) {

        (void) fseek(*prev_file_out_fp, begin - 1, SEEK_SET);

    }

    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, "Now starting decoding file at position %"G_GUINT64_FORMAT"\n", begin);

    crc32_init_ctx(&crc32_context);

    /* Start the real decoding */

    do {

        int len = fread(buf, 1, sizeof(buf), fp);

        int i;

        char buf_out[1024];

        int len_out = 0;

        memset(buf_out, 0, sizeof(buf_out));

        if (len == 0) {

            if (feof(fp)) {

                /* All data is read */

                break;

            }

        } else if (len < 0) {

            /* Some error occured while reading */

            *decoder_result = DECODER_RESULT_ERROR;

            if (errmsg) {

                *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to read from file '%s': %s"), __FILE__, __LINE__, filename_in, strerror(errno));

            }

            fclose(fp);

            return TRUE;

        }

        for (i = 0; i < len; i++) {

            unsigned char ch = buf[i];

            if (ch == '\r' || ch == '\n') {

                /* Ignore newlines */

                continue;

            } else if (ch == '=') {

                /* Escape character detected. Un-escape the next byte */

                escape_char_detected = TRUE;

            } else if (escape_char_detected && ch == 'y') {     // =y detected. This is probably the end of the part

                char buf_new[1024];

                int buf_new_length;

                char *ptr;

                guint32 crc32_expected = 0;

                memset(&buf_new, 0, sizeof(buf_new));

                buf_new_length = sizeof(buf) - i - 1;

                memcpy(buf_new, buf + i + 1, buf_new_length);

                /* Try to put more data in the buffer */

                if (fread(buf_new + buf_new_length, 1, sizeof(buf_new) - buf_new_length, fp) < 0) {

                    *decoder_result = DECODER_RESULT_ERROR;

                    if (errmsg) {

                        *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to read from file '%s': %s"), __FILE__, __LINE__, filename_in, strerror(errno));

                    }

                    fclose(fp);

                    return TRUE;

                }

                /* Do we really have an =yend ? */

                if (strncmp(buf_new, "end", 3) != 0) {

                    *decoder_result = DECODER_RESULT_INCOMPLETE;

                }

                /* Flush the buffer */

                if (len_out > 0 && fwrite(buf_out, 1, len_out, *prev_file_out_fp) <= 0) {

                    *decoder_result = DECODER_RESULT_ERROR;

                    if (errmsg) {

                        *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to write to file '%s': %s"), __FILE__, __LINE__, filename_out, strerror(errno));

                    }

                }

                fclose(fp);

#if 0 

                /* Update the MD5 */

                MD5Update(md5_ctx, (unsigned char*) buf_out, len_out);

#endif

                /* Calculate the CRC value of this part */

                crc32_process_bytes(buf_out, len_out, &crc32_context);

                /* Verify the CRC value of this part */

                if ((ptr = strstr(buf_new, "pcrc32="))) {

                    char crc[9];

                    memcpy(crc, ptr + 7, 8);

                    crc[8] = '\0';

                    crc32_expected = atocrc32(crc);

                } else if ((ptr = strstr(buf_new, "crc32="))) {

                    char crc[9];

                    memcpy(crc, ptr + 6, 8);

                    crc[8] = '\0';

                    crc32_expected = atocrc32(crc);

                } else {

                    /* No CRC32 value found. Ignore verification */

                    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_DEBUG, __FILE__ ":%i No CRC32 value found in file %s\n", __LINE__, filename_in);

                }

                if (crc32_expected != 0 && crc32_read_ctx(&crc32_context) != crc32_expected) {

                    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, "Part CRC32 error for file %s -- got 0x%08x, should be 0x%08x\n", filename_in, crc32_read_ctx(&crc32_context), crc32_expected);

                    *decoder_result = DECODER_RESULT_INCOMPLETE;

                }

                crc32_finish_ctx(&crc32_context);

                return TRUE;

            } else {

                ch = ch - 42 - (escape_char_detected ? 64 : 0);

                if (ch < 0) {

                    ch += 256;

                }

                buf_out[len_out++] = ch;

                g_return_val_if_fail(len_out <= sizeof(buf_out), TRUE);

                escape_char_detected = FALSE;

            }

        }

        /* Calculate the CRC and MD5 values of this part */

        crc32_process_bytes(buf_out, len_out, &crc32_context);

#if 0 

        MD5Update(md5_ctx, (unsigned char*) buf_out, len_out);

#endif

        /* Flush the buffer */

        if (fwrite(buf_out, 1, len_out, *prev_file_out_fp) <= 0) {

            *decoder_result = DECODER_RESULT_ERROR;

            if (errmsg) {

                *errmsg = g_strdup_printf(_("%s:%i The decoder wasn't able to write to file '%s': %s"), __FILE__, __LINE__, filename_out, strerror(errno));

            }

            fclose(fp);

            return TRUE;

        }

    } while (TRUE);

    fclose(fp);

    /* Unexpected end of file detected, file must be incomplete */

    *decoder_result = DECODER_RESULT_INCOMPLETE;

    return TRUE;

}

static NNTPGrabDecoderRes

nntpgrab_plugin_decoder_decode_file(NGPlugin *plugin_data, const char *collection_name, NNTPFile *file, const char *temp_directory, const char *target_directory, int *saved_errno, char **errmsg)

{

    int i;

#if 0 

    GList *lines = NULL;

#endif

    NNTPGrabDecoderRes ret = DECODER_RESULT_COMPLETE;

    gboolean file_found = FALSE;

    struct decoded_file *decoded_list = NULL;

    struct decoded_file *list;

    gboolean new_decoder_used = FALSE;

    char filename[1024];

    gboolean incomplete_file = FALSE;

    char target_path[4096];

    char prev_filename_out[1024];

    char prev_real_filename_out[1024];

    FILE *prev_file_fp = NULL;

    MD5_CTX md5_ctx;

    g_return_val_if_fail(collection_name != NULL, DECODER_RESULT_ERROR);

    g_return_val_if_fail(file != NULL, DECODER_RESULT_ERROR);

    g_return_val_if_fail(temp_directory != NULL, DECODER_RESULT_ERROR);

    g_return_val_if_fail(target_directory != NULL, DECODER_RESULT_ERROR);

    g_return_val_if_fail(saved_errno != NULL, DECODER_RESULT_ERROR);

    *saved_errno = 0;

    memset(prev_filename_out, 0, sizeof(prev_filename_out));

    memset(prev_real_filename_out, 0, sizeof(prev_real_filename_out));

    memset(target_path, 0, sizeof(target_path));

    g_snprintf(target_path, sizeof(target_path) - 1, "%s"G_DIR_SEPARATOR_S"%s", target_directory, collection_name);

    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("File '%s' of collection '%s' is now being decoded to directory '%s'"), file->subject, collection_name, target_path);

    if (g_mkdir_with_parents(target_path, 0755) == -1) {

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Creation of folder '%s' failed, errno = %i, %s\n"), target_path, errno, strerror(errno));

    }

    // The yydecode code assumes all the resulting files will be saved in the current directory

    // so we need to change to the wanted directory before starting the decode

    if (g_chdir(target_path) == -1) {

        if (errmsg) {

            *errmsg = g_strdup_printf(_("Unable to navigate to folder '%s': %s"), target_path, strerror(errno));

        }

        *saved_errno = ENOENT;

        ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Unable to navigate to folder '%s': %s"), target_path, strerror(errno));

        return DECODER_RESULT_ERROR;

    }

    MD5Init(&md5_ctx);

    // Load all the individual parts

    for (i = 0; i < file->num_parts; i++) {

        struct decoded_file *list;

        memset(filename, 0, sizeof(filename) - 1);

        g_snprintf(filename, sizeof(filename) - 1, "%s%s%s.%i", temp_directory, G_DIR_SEPARATOR_S, file->tmp_filename, i + 1);

        if (!g_file_test(filename, G_FILE_TEST_IS_REGULAR | G_FILE_TEST_EXISTS)) {

            continue;

        }

        file_found = TRUE;

        if (try_to_use_new_decoder(plugin_data, filename, target_path, &ret, errmsg, &prev_file_fp, prev_filename_out, sizeof(prev_filename_out) - 1, prev_real_filename_out, sizeof(prev_real_filename_out) - 1, &md5_ctx)) {

#if 0 

g_print("prev_file_fp = %#x\n", prev_file_fp);

g_print("prev_real_filename_out = %s\n", prev_real_filename_out);

g_print("ret = %i\n", ret);

#endif

            strncpy((char*) file->real_filename, prev_real_filename_out, sizeof(file->real_filename) - 1);

            new_decoder_used = TRUE;

            if (ret == DECODER_RESULT_ERROR) {

                g_print("Error during decode\n");

                if (errmsg && *errmsg) {

                    g_print("errmsg = %s\n", *errmsg);

                }

                break;

            } else if (ret == DECODER_RESULT_INCOMPLETE) {

                incomplete_file = TRUE;

            }

            continue;

        }

        if (decode(filename, NULL, &decoded_list, saved_errno, errmsg) != EXIT_SUCCESS) {

            g_print("Error during decode\n");

            if (errmsg && *errmsg) {

                g_print("errmsg = %s\n", *errmsg);

            }

            ret = DECODER_RESULT_ERROR;

            while (decoded_list) {

                list = decoded_list->next;

                if (decoded_list->handle) {

                    fclose(decoded_list->handle);

                    decoded_list->handle = NULL;

                }

                if (decoded_list->filename) free(decoded_list->filename);

                if (decoded_list->outname)  free(decoded_list->outname);

                if (decoded_list->status)   free(decoded_list->status);

                free(decoded_list);

                decoded_list = list;

            }

        }

        if (ret == DECODER_RESULT_ERROR) {

            break;

        }

    }

    if (new_decoder_used) {

        unsigned char md5_digest[16];

        int j;

        /* Close any file descriptors which may have left open */

        if (prev_file_fp) {

            if (fclose(prev_file_fp) != 0) {

                ret = DECODER_RESULT_ERROR;

                if (errmsg) {

                    *errmsg = g_strdup_printf(_("%s:%i Unable to close file '%s': %s"), __FILE__, __LINE__, prev_filename_out, strerror(errno));

                }

            }

            prev_file_fp = NULL;

        }

        MD5Final(md5_digest, &md5_ctx);

        memset(file->md5sum, 0, sizeof(file->md5sum));

        for (j = 0; j < 16; j++) {

            sprintf(file->md5sum + j * 2, "%02x", md5_digest[j]);

        }

        if (incomplete_file && ret != DECODER_RESULT_ERROR) {

            ret = DECODER_RESULT_INCOMPLETE;

        }

    } else {

        if (decoded_list) {

            strncpy((char*) file->real_filename, decoded_list->outname, sizeof(file->real_filename) - 1);

            for(i = 0; i < decoded_list->total_parts; i++) {

                switch(decoded_list->status[i]) {

                    case part_missing:

                    case part_broken:

                        ret = DECODER_RESULT_INCOMPLETE;

                        break;

                    case part_intact:

                    case part_duplicated:

                        // don't do anything

                        break;

                }

            }

            while (decoded_list) {

                list = decoded_list->next;

                if (decoded_list->handle) {

                    fclose(decoded_list->handle);

                    decoded_list->handle = NULL;

                }

                if (decoded_list->filename) free(decoded_list->filename);

                if (decoded_list->outname)  free(decoded_list->outname);

                if (decoded_list->status)   free(decoded_list->status);

                free(decoded_list);

                decoded_list = list;

            }

        }

    }

    if (!file_found) {

        if (errmsg) {

            *errmsg = g_strdup(_("No data to decode"));

        }

        if (saved_errno) {

            *saved_errno = -1;

        }

        ret = DECODER_RESULT_NO_PARTS_AVAIL;

    }

    if (ret != DECODER_RESULT_ERROR) {

        char filename[1024];

        // Remove all the temp parts on success

        for (i = 0; i < file->num_parts; i++) {

            memset(&filename, 0, sizeof(filename));

            g_snprintf(filename, sizeof(filename) - 1, "%s%s%s.%i", temp_directory, G_DIR_SEPARATOR_S, file->tmp_filename, i + 1);

            g_unlink(filename);

        }

    }

    ng_plugin_emit_log_msg(plugin_data, NG_LOG_LEVEL_INFO, _("Decoding completed with return value %i"), ret);

    // Change the current directory back to another directory to prevent the NNTPGrab process

    // from 'claiming' the directory in which the decoded files were saved

    g_chdir(target_directory);

    return ret;

}