Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge branch 'charset' into 'master'
Convert text parts to utf-8

See merge request mer-core/mms-engine!26
  • Loading branch information
monich committed Aug 6, 2020
2 parents 754e0ed + 2bda88f commit 699bf8d
Show file tree
Hide file tree
Showing 16 changed files with 393 additions and 98 deletions.
42 changes: 24 additions & 18 deletions mms-engine/main.c
@@ -1,7 +1,7 @@
/*
* Copyright (C) 2013-2020 Jolla Ltd.
* Copyright (C) 2013-2020 Slava Monich <slava.monich@jolla.com>
* Copyright (C) 2019 Open Mobile Platform LLC.
* Copyright (C) 2020 Open Mobile Platform LLC.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
Expand Down Expand Up @@ -403,18 +403,21 @@ mms_app_parse_options(
gboolean disable_dbus_log = FALSE;
gint size_limit_kb = -1;
gdouble megapixels = -1;
MMSConfig* config = &opt->global.config;
MMSSettingsSimDataCopy* settings = &opt->settings;

char* root_dir_help = g_strdup_printf(
"Root directory for MMS files [%s]",
opt->global.config.root_dir);
config->root_dir);
char* retry_secs_help = g_strdup_printf(
"Retry period in seconds [%d]",
opt->global.config.retry_secs);
config->retry_secs);
char* network_idle_secs_help = g_strdup_printf(
"Network inactivity timeout in seconds [%d]",
opt->global.config.network_idle_secs);
config->network_idle_secs);
char* idle_secs_help = g_strdup_printf(
"Service inactivity timeout in seconds [%d]",
opt->global.config.idle_secs);
config->idle_secs);
char* description = gutil_log_description(NULL, 0);

GOptionContext* options;
Expand All @@ -428,12 +431,12 @@ mms_app_parse_options(
{ "root-dir", 'd', 0, G_OPTION_ARG_FILENAME,
&root_dir, root_dir_help, "DIR" },
{ "retry-secs", 'r', 0, G_OPTION_ARG_INT,
&opt->global.config.retry_secs, retry_secs_help, "SEC" },
&config->retry_secs, retry_secs_help, "SEC" },
{ "network-idle-secs", 'n', 0, G_OPTION_ARG_INT,
&opt->global.config.network_idle_secs,
&config->network_idle_secs,
network_idle_secs_help, "SEC" },
{ "idle-secs", 'i', 0, G_OPTION_ARG_INT,
&opt->global.config.idle_secs, idle_secs_help, "SEC" },
&config->idle_secs, idle_secs_help, "SEC" },
{ "size-limit", 's', 0, G_OPTION_ARG_INT,
&size_limit_kb, "Maximum size for outgoing messages", "KB" },
{ "pix-limit", 'p', 0, G_OPTION_ARG_DOUBLE,
Expand All @@ -445,12 +448,15 @@ mms_app_parse_options(
{ "keep-running", 'k', 0, G_OPTION_ARG_NONE, &keep_running,
"Keep running after everything is done", NULL },
{ "keep-temp-files", 't', 0, G_OPTION_ARG_NONE,
&opt->global.config.keep_temp_files,
&config->keep_temp_files,
"Don't delete temporary files", NULL },
{ "preserve-encoding", 'e', G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE,
&config->convert_to_utf8,
"Don't convert text parts to UTF-8", NULL },
{ "attic", 'a', 0, G_OPTION_ARG_NONE,
&opt->global.config.attic_enabled,
&config->attic_enabled,
"Store unrecognized push messages in the attic", NULL },
#define OPT_VERBOSE_INDEX 13
#define OPT_VERBOSE_INDEX 14
{ "verbose", 'v', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
mms_app_option_verbose, "Be verbose (equivalent to -l=verbose)",
NULL },
Expand Down Expand Up @@ -560,28 +566,28 @@ mms_app_parse_options(
GINFO("Starting");
#endif
if (size_limit_kb >= 0) {
opt->settings.data.size_limit = size_limit_kb * 1024;
settings->data.size_limit = size_limit_kb * 1024;
opt->flags |= MMS_ENGINE_FLAG_OVERRIDE_SIZE_LIMIT;
}
if (megapixels >= 0) {
opt->settings.data.max_pixels = (int)(megapixels*1000)*1000;
settings->data.max_pixels = (int)(megapixels*1000)*1000;
opt->flags |= MMS_ENGINE_FLAG_OVERRIDE_MAX_PIXELS;
}
if (ua) {
g_free(opt->settings.user_agent);
opt->settings.data.user_agent = opt->settings.user_agent = ua;
g_free(settings->user_agent);
settings->data.user_agent = settings->user_agent = ua;
opt->flags |= MMS_ENGINE_FLAG_OVERRIDE_USER_AGENT;
ua = NULL;
}
if (uaprof) {
g_free(opt->settings.uaprof);
opt->settings.data.uaprof = opt->settings.uaprof = uaprof;
g_free(settings->uaprof);
settings->data.uaprof = settings->uaprof = uaprof;
opt->flags |= MMS_ENGINE_FLAG_OVERRIDE_UAPROF;
uaprof = NULL;
}
if (root_dir) {
g_free(opt->global.root_dir);
opt->global.config.root_dir = opt->global.root_dir = root_dir;
config->root_dir = opt->global.root_dir = root_dir;
root_dir = NULL;
}
if (keep_running) opt->flags |= MMS_ENGINE_FLAG_KEEP_RUNNING;
Expand Down
1 change: 1 addition & 0 deletions mms-lib/Makefile
Expand Up @@ -28,6 +28,7 @@ SRC = \
mms_attachment.c \
mms_attachment_image.c \
mms_attachment_jpeg.c \
mms_attachment_text.c \
mms_codec.c \
mms_connection.c \
mms_connman.c \
Expand Down
13 changes: 7 additions & 6 deletions mms-lib/include/mms_settings.h
@@ -1,7 +1,7 @@
/*
* Copyright (C) 2014-2019 Jolla Ltd.
* Copyright (C) 2014-2019 Slava Monich <slava.monich@jolla.com>
* Copyright (C) 2019 Open Mobile Platform LLC.
* Copyright (C) 2014-2020 Jolla Ltd.
* Copyright (C) 2014-2020 Slava Monich <slava.monich@jolla.com>
* Copyright (C) 2020 Open Mobile Platform LLC.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
Expand All @@ -13,8 +13,8 @@
* GNU General Public License for more details.
*/

#ifndef JOLLA_MMS_SETTINGS_H
#define JOLLA_MMS_SETTINGS_H
#ifndef SAILFISH_MMS_SETTINGS_H
#define SAILFISH_MMS_SETTINGS_H

#include "mms_lib_types.h"

Expand All @@ -24,6 +24,7 @@ struct mms_config {
int retry_secs; /* Retry timeout in seconds */
int network_idle_secs; /* Network inactivity timeout */
int idle_secs; /* Service inactivity timeout */
gboolean convert_to_utf8; /* Convert text parts to UTF-8 */
gboolean keep_temp_files; /* Keep temporary files around */
gboolean attic_enabled; /* Keep unrecognized push message in attic */
};
Expand Down Expand Up @@ -145,7 +146,7 @@ mms_settings_sim_data_copy(
#define mms_settings_sim_data_reset(data) \
mms_settings_sim_data_copy(data,NULL)

#endif /* JOLLA_MMS_SETTINGS_H */
#endif /* SAILFISH_MMS_SETTINGS_H */

/*
* Local Variables:
Expand Down
1 change: 1 addition & 0 deletions mms-lib/mms-lib.pro
Expand Up @@ -28,6 +28,7 @@ SOURCES += \
src/mms_attachment.c \
src/mms_attachment_image.c \
src/mms_attachment_jpeg.c \
src/mms_attachment_text.c \
src/mms_attachment_qt.cpp \
src/mms_codec.c \
src/mms_connection.c \
Expand Down
125 changes: 93 additions & 32 deletions mms-lib/src/mms_attachment.c
@@ -1,6 +1,7 @@
/*
* Copyright (C) 2013-2016 Jolla Ltd.
* Contact: Slava Monich <slava.monich@jolla.com>
* Copyright (C) 2013-2020 Jolla Ltd.
* Copyright (C) 2013-2020 Slava Monich <slava.monich@jolla.com>
* Copyright (C) 2020 Open Mobile Platform LLC.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
Expand All @@ -17,6 +18,8 @@
#include "mms_settings.h"
#include "mms_codec.h"

#include "gutil_strv.h"

#ifdef HAVE_MAGIC
# include <magic.h>
#endif
Expand All @@ -27,7 +30,6 @@
GLOG_MODULE_DEFINE("mms-attachment");

#define MMS_ATTACHMENT_DEFAULT_TYPE "application/octet-stream"
#define CONTENT_TYPE_PARAM_NAME "name"

G_DEFINE_TYPE(MMSAttachment, mms_attachment, G_TYPE_OBJECT)

Expand Down Expand Up @@ -58,8 +60,7 @@ mms_attachment_finalize(
MMSAttachment* at = MMS_ATTACHMENT(object);
GVERBOSE_("%p", at);
if (at->map) g_mapped_file_unref(at->map);
if (!at->config->keep_temp_files &&
!(at->flags & MMS_ATTACHMENT_KEEP_FILES)) {
if (!at->config->keep_temp_files) {
char* dir = g_path_get_dirname(at->original_file);
remove(at->original_file);
rmdir(dir);
Expand Down Expand Up @@ -201,6 +202,7 @@ mms_attachment_write_smil(
return FALSE;
}

static
char*
mms_attachment_guess_content_type(
const char* path)
Expand Down Expand Up @@ -239,6 +241,38 @@ mms_attachment_guess_content_type(
return content_type;
}

static
char*
mms_attachment_guess_text_encoding(
const char* path)
{
char* encoding = NULL;
const char* detected = NULL;

#ifdef HAVE_MAGIC
/* Use magic to determine mime type */
magic_t magic = magic_open(MAGIC_MIME_ENCODING);
if (magic) {
if (magic_load(magic, NULL) == 0) {
detected = magic_file(magic, path);
}
}
#endif

if (detected) {
GDEBUG("%s: detected %s", path, detected);
encoding = g_strdup(detected);
} else {
encoding = g_strdup(MMS_DEFAULT_CHARSET);
}

#ifdef HAVE_MAGIC
if (magic) magic_close(magic);
#endif

return encoding;
}

MMSAttachment*
mms_attachment_new_smil(
const MMSConfig* config,
Expand All @@ -259,7 +293,8 @@ mms_attachment_new_smil(
if (ok) {
MMSAttachmentInfo ai;
ai.file_name = path;
ai.content_type = SMIL_CONTENT_TYPE "; charset=utf-8";
ai.content_type = SMIL_CONTENT_TYPE "; "
CONTENT_TYPE_PARAM_CHARSET "=" MMS_DEFAULT_CHARSET;
ai.content_id = NULL;
smil = mms_attachment_new(config, &ai, error);
GASSERT(smil && (smil->flags & MMS_ATTACHMENT_SMIL));
Expand Down Expand Up @@ -290,30 +325,51 @@ mms_attachment_new(
char* media_type = NULL;
char* name = g_path_get_basename(path);
char* content_type = NULL;
GType type;
GType type = MMS_TYPE_ATTACHMENT;
MMSAttachment* at;

MMSAttachmentClass* klass;

/*
* We always need to provide charset for text attachments because
* operators may (and often do) want to convert those to their
* favorite charset, in which case they need to know the original
* one (some just blindly assume us-ascii and mess things up).
*/
if (info->content_type && info->content_type[0]) {
char** ct = mms_parse_http_content_type(info->content_type);
if (ct) {
char** ptr;
gboolean append_name = TRUE;
if (!strcmp(ct[0], SMIL_CONTENT_TYPE)) {
flags |= MMS_ATTACHMENT_SMIL;
}
gboolean append_charset = g_str_has_prefix(ct[0],
MEDIA_TYPE_TEXT_PREFIX);

for (ptr = ct+1; *ptr; ptr+=2) {
if (!g_ascii_strcasecmp(ptr[0], CONTENT_TYPE_PARAM_NAME)) {
const char* p = ptr[0];

if (!g_ascii_strcasecmp(p, CONTENT_TYPE_PARAM_NAME)) {
g_free(ptr[1]);
ptr[1] = g_strdup(name);
append_name = FALSE;
} else if (append_charset && !g_ascii_strcasecmp(p,
CONTENT_TYPE_PARAM_CHARSET)) {
/* Charset is provided by the caller */
append_charset = FALSE;
}
}
if (append_name) {
const guint len = g_strv_length(ct);
ct = g_renew(gchar*, ct, len+3);
ct[len] = g_strdup(CONTENT_TYPE_PARAM_NAME);
ct[len+1] = g_strdup(name);
ct[len+2] = NULL;
ct = gutil_strv_addv(ct,
CONTENT_TYPE_PARAM_NAME, name,
NULL);
}
if (append_charset) {
char* enc = mms_attachment_guess_text_encoding(path);

if (enc) {
ct = gutil_strv_addv(ct,
CONTENT_TYPE_PARAM_CHARSET, enc,
NULL);
g_free(enc);
}
}
content_type = mms_unparse_http_content_type(ct);
media_type = g_strdup(ct[0]);
Expand All @@ -322,40 +378,37 @@ mms_attachment_new(
}

if (!content_type) {
const char* default_charset = "utf-8";
char* detected = NULL;
const char* charset = NULL;
const char* ct[6];
int n = 0;

media_type = mms_attachment_guess_content_type(path);
if (!strcmp(media_type, SMIL_CONTENT_TYPE)) {
flags |= MMS_ATTACHMENT_SMIL;
charset = default_charset;
} else {
if (g_str_has_prefix(media_type, MEDIA_TYPE_TEXT_PREFIX)) {
charset = default_charset;
}
if (g_str_has_prefix(media_type, MEDIA_TYPE_TEXT_PREFIX)) {
detected = mms_attachment_guess_text_encoding(path);
charset = detected;
}

ct[n++] = media_type;
if (charset) {
ct[n++] = "charset";
ct[n++] = CONTENT_TYPE_PARAM_CHARSET;
ct[n++] = charset;
}
ct[n++] = CONTENT_TYPE_PARAM_NAME;
ct[n++] = name;
ct[n++] = NULL;
content_type = mms_unparse_http_content_type((char**)ct);
g_free(detected);
}

GDEBUG("%s: %s", path, content_type);

if (!strcmp(media_type, "image/jpeg")) {
if (!strcmp(media_type, SMIL_CONTENT_TYPE)) {
flags |= MMS_ATTACHMENT_SMIL;
} else if (!strcmp(media_type, "image/jpeg")) {
type = MMS_TYPE_ATTACHMENT_JPEG;
} else if (g_str_has_prefix(media_type, MEDIA_TYPE_IMAGE_PREFIX)) {
type = MMS_TYPE_ATTACHMENT_IMAGE;
} else {
type = MMS_TYPE_ATTACHMENT;
} else if (g_str_has_prefix(media_type, MEDIA_TYPE_TEXT_PREFIX)) {
type = MMS_TYPE_ATTACHMENT_TEXT;
}

at = g_object_new(type, NULL);
Expand All @@ -369,8 +422,16 @@ mms_attachment_new(
g_strdup(info->content_id) :
g_strdup(at->content_location);

path = NULL;
g_free(media_type);
return at;
klass = MMS_ATTACHMENT_GET_CLASS(at);

if (!klass->fn_init || klass->fn_init(at)) {
GDEBUG("%s: %s", at->file_name, at->content_type);
return at;
}
/* Init failed */
mms_attachment_unref(at);
}
g_free(path);
}
Expand Down

0 comments on commit 699bf8d

Please sign in to comment.