Commit 53d6bb5d authored by Rémi Denis-Courmont's avatar Rémi Denis-Courmont

Use UTF-8 internally (closes #16 and introduces lots of yet to be uncovered bugs)

* libvlc: Conversion functions (needs performance improvement)
          Convert filenames from command line
* SAP, Skins2: no perform own UTF-8 conversion
* file, playlistold : convert
* wx, winCE : assume UTF-8
parent 57c60ce8
......@@ -228,14 +228,6 @@ fi
AM_CONDITIONAL(BUILD_INTL, test "${nls_cv_force_use_gnu_gettext}" = "yes")
XGETTEXT="${XGETTEXT} --keyword=_NS --keyword=_ANS"
AC_ARG_ENABLE(utf8,
[ --enable-utf8 unicode utf8 support (default enabled on darwin/beos/win32)])
if test "${enable_utf8}" = "yes" || (test "${enable_utf8}" != "no" && (test "${SYS}" = "mingw32" || test "${SYS}" = "beos" || test "${SYS}" = "darwin" ) )
then
AC_DEFINE(ENABLE_UTF8, 1,
Define if you want utf8 support)
fi
dnl
dnl Iconv stuff
dnl
......
......@@ -26,6 +26,10 @@ extern "C" {
# endif
VLC_EXPORT( vlc_bool_t, vlc_current_charset, ( char ** ) );
VLC_EXPORT( void, LocaleFree, ( const char * ) );
VLC_EXPORT( char *, FromLocale, ( const char * ) );
VLC_EXPORT( char *, ToLocale, ( const char * ) );
VLC_EXPORT( char *, EnsureUTF8, ( char * ) );
# ifdef __cplusplus
}
......
......@@ -292,7 +292,6 @@ struct sap_handler_t
int (*pf_del) ( sap_handler_t*, session_descriptor_t *);
/* private data, not in p_sys as there is one kind of sap_handler_t */
vlc_iconv_t iconvHandle;
};
/* The main announce handler object */
......
......@@ -383,6 +383,10 @@ struct module_symbols_t
int (*ACL_AddNet_inner) (vlc_acl_t *p_acl, const char *psz_ip, int i_len, vlc_bool_t b_allow);
void (*ACL_Destroy_inner) (vlc_acl_t *p_acl);
char* (*httpd_ServerIP_inner) (httpd_client_t *cl, char *psz_ip);
char * (*FromLocale_inner) (const char *);
void (*LocaleFree_inner) (const char *);
char * (*ToLocale_inner) (const char *);
char * (*EnsureUTF8_inner) (char *);
};
# if defined (__PLUGIN__)
# define aout_FiltersCreatePipeline (p_symbols)->aout_FiltersCreatePipeline_inner
......@@ -751,6 +755,10 @@ struct module_symbols_t
# define ACL_AddNet (p_symbols)->ACL_AddNet_inner
# define ACL_Destroy (p_symbols)->ACL_Destroy_inner
# define httpd_ServerIP (p_symbols)->httpd_ServerIP_inner
# define FromLocale (p_symbols)->FromLocale_inner
# define LocaleFree (p_symbols)->LocaleFree_inner
# define ToLocale (p_symbols)->ToLocale_inner
# define EnsureUTF8 (p_symbols)->EnsureUTF8_inner
# elif defined (HAVE_DYNAMIC_PLUGINS) && !defined (__BUILTIN__)
/******************************************************************
* STORE_SYMBOLS: store VLC APIs into p_symbols for plugin access.
......@@ -1122,6 +1130,10 @@ struct module_symbols_t
((p_symbols)->ACL_AddNet_inner) = ACL_AddNet; \
((p_symbols)->ACL_Destroy_inner) = ACL_Destroy; \
((p_symbols)->httpd_ServerIP_inner) = httpd_ServerIP; \
((p_symbols)->FromLocale_inner) = FromLocale; \
((p_symbols)->LocaleFree_inner) = LocaleFree; \
((p_symbols)->ToLocale_inner) = ToLocale; \
((p_symbols)->EnsureUTF8_inner) = EnsureUTF8; \
(p_symbols)->net_ConvertIPv4_deprecated = NULL; \
# endif /* __PLUGIN__ */
......
......@@ -69,6 +69,8 @@
# define lseek fseek
#endif
#include "charset.h"
/*****************************************************************************
* Module descriptor
*****************************************************************************/
......@@ -106,7 +108,7 @@ static int Seek( access_t *, int64_t );
static int Read( access_t *, uint8_t *, int );
static int Control( access_t *, int, va_list );
static int _OpenFile( access_t *, char * );
static int _OpenFile( access_t *, const char * );
typedef struct
{
......@@ -573,12 +575,16 @@ static int Control( access_t *p_access, int i_query, va_list args )
/*****************************************************************************
* OpenFile: Opens a specific file
*****************************************************************************/
static int _OpenFile( access_t * p_access, char * psz_name )
static int _OpenFile( access_t * p_access, const char * psz_name )
{
access_sys_t *p_sys = p_access->p_sys;
const char *psz_localname;
psz_localname = ToLocale( psz_name );
#ifdef UNDER_CE
p_sys->fd = fopen( psz_name, "rb" );
p_sys->fd = fopen( psz_localname, "rb" );
LocaleFree( psz_localname );
if ( !p_sys->fd )
{
msg_Err( p_access, "cannot open file %s", psz_name );
......@@ -591,7 +597,8 @@ static int _OpenFile( access_t * p_access, char * psz_name )
fseek( p_sys->fd, 0, SEEK_SET );
#else
p_sys->fd = open( psz_name, O_NONBLOCK /*| O_LARGEFILE*/ );
p_sys->fd = open( psz_localname, O_NONBLOCK /*| O_LARGEFILE*/ );
LocaleFree( psz_localname );
if ( p_sys->fd == -1 )
{
msg_Err( p_access, "cannot open file %s (%s)", psz_name,
......
......@@ -26,35 +26,17 @@
#include "playlist.hpp"
#include "../utils/ustring.hpp"
#include "charset.h"
Playlist::Playlist( intf_thread_t *pIntf ): VarList( pIntf )
{
// Get the playlist VLC object
m_pPlaylist = pIntf->p_sys->p_playlist;
// Try to guess the current charset
char *pCharset;
vlc_current_charset( &pCharset );
iconvHandle = vlc_iconv_open( "UTF-8", pCharset );
msg_Dbg( pIntf, "Using character encoding: %s", pCharset );
#ifndef WIN32
// vlc_current_charset returns a pointer on a satic char[] on win32
free( pCharset );
#endif
if( iconvHandle == (vlc_iconv_t)-1 )
{
msg_Warn( pIntf, "Unable to do requested conversion" );
}
buildList();
}
Playlist::~Playlist()
{
if( iconvHandle != (vlc_iconv_t)-1 ) vlc_iconv_close( iconvHandle );
}
......@@ -114,7 +96,7 @@ void Playlist::buildList()
for( int i = 0; i < m_pPlaylist->i_size; i++ )
{
// Get the name of the playlist item
UString *pName = convertName( m_pPlaylist->pp_items[i]->input.psz_name );
UString *pName = m_pPlaylist->pp_items[i]->input.psz_name;
// Is it the played stream ?
bool playing = (i == m_pPlaylist->i_index );
// Add the item in the list
......@@ -126,37 +108,6 @@ void Playlist::buildList()
UString *Playlist::convertName( const char *pName )
{
if( iconvHandle == (vlc_iconv_t)-1 )
{
return new UString( getIntf(), pName );
}
char *pNewName, *pBufferOut;
const char *pBufferIn;
size_t ret, inbytesLeft, outbytesLeft;
// Try to convert the playlist item into UTF8
pNewName = (char*)malloc( 6 * strlen( pName ) );
pBufferOut = pNewName;
pBufferIn = pName;
inbytesLeft = strlen( pName );
outbytesLeft = 6 * inbytesLeft;
// ICONV_CONST is defined in config.h
ret = vlc_iconv( iconvHandle, (char **)&pBufferIn, &inbytesLeft,
&pBufferOut, &outbytesLeft );
*pBufferOut = '\0';
if( inbytesLeft )
{
msg_Warn( getIntf(), "Failed to convert the playlist item into UTF8" );
free( pNewName );
return new UString( getIntf(), pName );
}
else
{
UString *pString = new UString( getIntf(), pNewName );
free( pNewName );
return pString;
}
return new UString( getIntf(), pName );
}
......@@ -46,14 +46,9 @@ class Playlist: public VarList
private:
/// VLC playlist object
playlist_t *m_pPlaylist;
/// Iconv handle
vlc_iconv_t iconvHandle;
/// Build the list from the VLC playlist
void buildList();
/// Convert a string to UTF8 from the current encoding
UString *convertName( const char *pName );
};
......
......@@ -535,12 +535,6 @@ protected:
# define _TOMB(a) a
#endif
#if defined( ENABLE_NLS ) && defined( ENABLE_UTF8 )
# define ISUTF8 1
#else // ENABLE_NLS && ENABLE_UTF8
# define ISUTF8 0
#endif
/*****************************************************************************
* Misc definitions (mainly from aygshell.h)
*****************************************************************************/
......
......@@ -382,7 +382,7 @@ PrefsTreeCtrl::PrefsTreeCtrl( wxWindow *_p_parent, intf_thread_t *_p_intf,
if( psz_help )
{
config_data->psz_help = wraptext( strdup( psz_help ),
72 , ISUTF8 );
72 , 1 );
}
else
{
......@@ -430,7 +430,7 @@ PrefsTreeCtrl::PrefsTreeCtrl( wxWindow *_p_parent, intf_thread_t *_p_intf,
if( psz_help )
{
cd->psz_help = wraptext( strdup( psz_help ),72 ,
ISUTF8 );
1 );
}
else
{
......@@ -447,7 +447,7 @@ PrefsTreeCtrl::PrefsTreeCtrl( wxWindow *_p_parent, intf_thread_t *_p_intf,
if( psz_help )
{
config_data->psz_help = wraptext( strdup( psz_help ) ,
72 , ISUTF8 );
72 , 1 );
}
else
{
......
......@@ -440,7 +440,7 @@ ModuleListCatConfigControl::ModuleListCatConfigControl( vlc_object_t *p_this,
text->SetToolTip( wxU(p_item->psz_longtext) );
sizer->Add(text, 0, wxEXPAND|wxALL, 5 );
sizer->Add (new wxStaticText( this, -1, wxU( vlc_wraptext( _("Select modules that you want. To get more advanced control, you can also modify the resulting chain by yourself") , 72, ISUTF8 ) ) ) );
sizer->Add (new wxStaticText( this, -1, wxU( vlc_wraptext( _("Select modules that you want. To get more advanced control, you can also modify the resulting chain by yourself") , 72, 1 ) ) ) );
sizer->Layout();
this->SetSizerAndFit( sizer );
......
......@@ -57,22 +57,18 @@ DECLARE_LOCAL_EVENT_TYPE( wxEVT_INTF, 1 );
#define SLIDER_MAX_POS 10000
/* wxU is used to convert ansi/utf8 strings to unicode strings (wchar_t) */
#if defined( ENABLE_NLS ) && defined( ENABLE_UTF8 )
#if defined( ENABLE_NLS )
#if wxUSE_UNICODE
# define wxU(utf8) wxString(utf8, wxConvUTF8)
#else
# define wxU(utf8) wxString(wxConvUTF8.cMB2WC(utf8), *wxConvCurrent)
#endif
#define ISUTF8 1
#else // ENABLE_NLS && ENABLE_UTF8
#else // ENABLE_NLS
#if wxUSE_UNICODE
# define wxU(ansi) wxString(ansi, wxConvLocal)
#else
# define wxU(ansi) (ansi)
#endif
#define ISUTF8 0
#endif
/* wxL2U (locale to unicode) is used to convert ansi strings to unicode
......
......@@ -54,8 +54,11 @@ int Export_Old( vlc_object_t *p_this )
for ( i = 0 ; i < p_playlist->i_size ; i++ )
{
fprintf( p_export->p_file , "%s\n" ,
p_playlist->pp_items[i]->input.psz_uri );
char *psz_uri;
psz_uri = ToLocale( p_playlist->pp_items[i]->input.psz_uri );
fprintf( p_export->p_file , "%s\n" , psz_uri );
LocaleFree( psz_uri );
}
return VLC_SUCCESS;
}
......@@ -215,9 +215,6 @@ struct services_discovery_sys_t
playlist_item_t *p_node;
playlist_t *p_playlist;
/* charset conversion */
vlc_iconv_t iconvHandle;
/* Table of announces */
int i_announces;
struct sap_announce_t **pp_announces;
......@@ -257,8 +254,6 @@ struct demux_sys_t
/* Helper functions */
static char *GetAttribute( sdp_t *p_sdp, const char *psz_search );
static vlc_bool_t IsSameSession( sdp_t *p_sdp1, sdp_t *p_sdp2 );
static char *convert_from_utf8( struct services_discovery_t *p_sd,
char *psz_unicode );
static int InitSocket( services_discovery_t *p_sd, char *psz_address, int i_port );
#ifdef HAVE_ZLIB_H
static int Decompress( unsigned char *psz_src, unsigned char **_dst, int i_len );
......@@ -278,20 +273,11 @@ static int Open( vlc_object_t *p_this )
malloc( sizeof( services_discovery_sys_t ) );
playlist_view_t *p_view;
char *psz_addr, *psz_charset;
char *psz_addr;
vlc_value_t val;
p_sys->i_timeout = var_CreateGetInteger( p_sd, "sap-timeout" );
vlc_current_charset( &psz_charset );
p_sys->iconvHandle = vlc_iconv_open( psz_charset, "UTF-8" );
free( psz_charset );
if( p_sys->iconvHandle == (vlc_iconv_t)(-1) )
{
msg_Err( p_sd, "unable to perform characters conversion" );
return VLC_EGENERIC;
}
p_sd->pf_run = Run;
p_sd->p_sys = p_sys;
......@@ -491,8 +477,6 @@ static void Close( vlc_object_t *p_this )
vlc_object_release( p_sys->p_playlist );
}
vlc_iconv_close( p_sys->iconvHandle );
free( p_sys );
}
......@@ -782,25 +766,21 @@ sap_announce_t *CreateAnnounce( services_discovery_t *p_sd, uint16_t i_hash,
sdp_t *p_sdp )
{
playlist_item_t *p_item, *p_child;
char *psz_value;
char *psz_value;
sap_announce_t *p_sap = (sap_announce_t *)malloc(
sizeof(sap_announce_t ) );
psz_value = convert_from_utf8( p_sd, p_sdp->psz_sessionname );
if( p_sap == NULL || psz_value == NULL )
{
FREE( p_sap );
FREE( psz_value );
if( p_sap == NULL )
return NULL;
}
EnsureUTF8( p_sdp->psz_sessionname );
p_sap->i_last = mdate();
p_sap->i_hash = i_hash;
p_sap->p_sdp = p_sdp;
p_sap->i_item_id = -1;
/* Create the playlist item here */
p_item = playlist_ItemNew( p_sd, p_sap->p_sdp->psz_uri, psz_value );
free( psz_value );
p_item = playlist_ItemNew( p_sd, p_sap->p_sdp->psz_uri,
p_sdp->psz_sessionname );
if( !p_item )
{
......@@ -829,27 +809,16 @@ sap_announce_t *CreateAnnounce( services_discovery_t *p_sd, uint16_t i_hash,
if( psz_value != NULL )
{
char *psz_grp = convert_from_utf8( p_sd, psz_value );
EnsureUTF8( psz_value );
if( psz_grp != NULL )
{
p_child = playlist_ChildSearchName( p_sd->p_sys->p_node,
psz_grp );
p_child = playlist_ChildSearchName( p_sd->p_sys->p_node, psz_value );
if( p_child == NULL )
{
p_child = playlist_NodeCreate( p_sd->p_sys->p_playlist,
VIEW_CATEGORY, psz_grp,
p_sd->p_sys->p_node );
p_child->i_flags &= ~PLAYLIST_SKIP_FLAG;
}
free( psz_grp );
}
else
if( p_child == NULL )
{
msg_Err( p_sd, "out of memory");
free( p_sap );
return NULL;
p_child = playlist_NodeCreate( p_sd->p_sys->p_playlist,
VIEW_CATEGORY, psz_value,
p_sd->p_sys->p_node );
p_child->i_flags &= ~PLAYLIST_SKIP_FLAG;
}
}
else
......@@ -1252,58 +1221,6 @@ static sdp_t * ParseSDP( vlc_object_t *p_obj, char* psz_sdp )
return p_sdp;
}
static char *convert_from_utf8( struct services_discovery_t *p_sd,
char *psz_unicode )
{
char *psz_local, *psz_in, *psz_out;
size_t ret, i_in, i_out;
vlc_bool_t b_warn = VLC_FALSE;
if( psz_unicode == NULL )
return NULL;
psz_in = psz_unicode;
i_in = strlen( psz_unicode );
#ifndef MB_CUR_MAX
i_out = 6 * i_in;
#else
i_out = MB_CUR_MAX * i_in;
#endif
psz_local = malloc( i_out + 1 );
if( psz_local == NULL )
return NULL;
psz_out = psz_local;
do
{
ret = vlc_iconv( p_sd->p_sys->iconvHandle,
&psz_in, &i_in, &psz_out, &i_out);
if( i_in )
{
*psz_in = '?';
b_warn = VLC_TRUE;
}
else
if( ret == (size_t)(-1) )
{
msg_Err( p_sd, "character conversion failure : %s",
strerror( errno ) );
free( psz_local );
return NULL;
}
}
while( i_in );
if( b_warn )
msg_Warn( p_sd, "in \"%s\" : %s", psz_unicode,
strerror( errno ) );
*psz_out = '\0';
return psz_local;
}
static int InitSocket( services_discovery_t *p_sd, char *psz_address,
int i_port )
{
......
......@@ -78,6 +78,7 @@
#include "video_output.h"
#include "stream_output.h"
#include "charset.h"
#include "libvlc.h"
......@@ -1878,11 +1879,111 @@ static void SetLanguage ( char const *psz_lang )
/* Set the default domain */
textdomain( PACKAGE_NAME );
#if defined( ENABLE_UTF8 )
bind_textdomain_codeset( PACKAGE_NAME, "UTF-8" );
#endif
}
#endif
/*****************************************************************************
* FromLocale: converts a locale string to UTF-8
*****************************************************************************/
/* FIXME FIXME: it really has to be made quicker */
char *FromLocale( const char *locale )
{
char *psz_charset;
if( !vlc_current_charset( &psz_charset ) )
{
char *iptr = (ICONV_CONST char *)locale, *output, *optr;
size_t inb, outb;
/* cannot fail (unless vlc_current_charset sucks) */
vlc_iconv_t hd = vlc_iconv_open( "UTF-8", psz_charset );
free( psz_charset );
/*
* We are not allowed to modify the locale pointer, even if we cast it to
* non-const.
*/
inb = strlen( locale );
outb = inb * 6 + 1;
/* FIXME: I'm not sure about the value for the multiplication
* (for western people, multiplication by 3 (Latin9) is sufficient) */
optr = output = calloc( outb , 1);
while( vlc_iconv( hd, &iptr, &inb, &optr, &outb ) == (size_t)-1 )
*iptr = '?'; /* should not happen, and yes, it sucks */
vlc_iconv_close( hd );
return realloc( output, strlen( output ) + 1 );
}
free( psz_charset );
return (char *)locale;
}
/*****************************************************************************
* ToLocale: converts an UTF-8 string to locale
*****************************************************************************/
/* FIXME FIXME: it really has to be made quicker */
char *ToLocale( const char *utf8 )
{
char *psz_charset;
if( !vlc_current_charset( &psz_charset ) )
{
char *iptr = (ICONV_CONST char *)utf8, *output, *optr;
size_t inb, outb;
/* cannot fail (unless vlc_current_charset sucks) */
vlc_iconv_t hd = vlc_iconv_open( psz_charset, "UTF-8" );
free( psz_charset );
/*
* We are not allowed to modify the locale pointer, even if we cast it to
* non-const.
*/
inb = strlen( utf8 );
/* FIXME: I'm not sure about the value for the multiplication
* (for western people, multiplication is not needed) */
outb = inb * 2 + 1;
optr = output = calloc( outb, 1 );
while( vlc_iconv( hd, &iptr, &inb, &optr, &outb ) == (size_t)-1 )
*iptr = '?'; /* should not happen, and yes, it sucks */
vlc_iconv_close( hd );
return realloc( output, strlen( output ) + 1 );
}
free( psz_charset );
return (char *)utf8;
}
void LocaleFree( const char *str )
{
/* FIXME: this deserve a price for the most inefficient peice of code */
char *psz_charset;
if( !vlc_current_charset( &psz_charset ) )
free( (char *)str );
free( psz_charset );
}
/* FIXME: don't use iconv at all */
char *EnsureUTF8( char *str )
{
vlc_iconv_t hd;
size_t inb, outb;
char *ostr, *istr;
ostr = istr = str;
inb = outb = strlen( str );
hd = vlc_iconv_open( "UTF-8", "UTF-8" );
while( vlc_iconv( hd, &istr, &inb, &ostr, &outb ) == (size_t)-1 )
*istr = '?';
vlc_iconv_close( hd );
return str;
}
/*****************************************************************************
......@@ -1899,6 +2000,7 @@ static int GetFilenames( vlc_t *p_vlc, int i_argc, char *ppsz_argv[] )
* and their input options */
for( i_opt = i_argc - 1; i_opt >= optind; i_opt-- )
{
const char *psz_target;
i_options = 0;
/* Count the input options */
......@@ -1910,10 +2012,13 @@ static int GetFilenames( vlc_t *p_vlc, int i_argc, char *ppsz_argv[] )
/* TODO: write an internal function of this one, to avoid
* unnecessary lookups. */
VLC_AddTarget( p_vlc->i_object_id, ppsz_argv[ i_opt ],
/* FIXME: should we convert options to UTF-8 as well ?? */
psz_target = FromLocale( ppsz_argv[ i_opt ] );
VLC_AddTarget( p_vlc->i_object_id, psz_target,
(char const **)( i_options ? &ppsz_argv[i_opt + 1] :
NULL ), i_options,
PLAYLIST_INSERT, 0 );
LocaleFree( psz_target );
}
return VLC_SUCCESS;
......
......@@ -98,7 +98,6 @@ static int announce_SAPAnnounceAdd( sap_handler_t *p_sap,
static int announce_SAPAnnounceDel( sap_handler_t *p_sap,
session_descriptor_t *p_session );
static char *convert_to_utf8( struct sap_handler_t *p_this, char *psz_local );
#define FREE( p ) if( p ) { free( p ); (p) = NULL; }
......@@ -112,7 +111,6 @@ static char *convert_to_utf8( struct sap_handler_t *p_this, char *psz_local );
sap_handler_t *announce_SAPHandlerCreate( announce_handler_t *p_announce )
{
sap_handler_t *p_sap;
char *psz_charset;
p_sap = vlc_object_create( p_announce, sizeof( sap_handler_t ) );
......@@ -124,12 +122,6 @@ sap_handler_t *announce_SAPHandlerCreate( announce_handler_t *p_announce )
vlc_mutex_init( p_sap, &p_sap->object_lock );
vlc_current_charset( &psz_charset );
p_sap->iconvHandle = vlc_iconv_open( "UTF-8", psz_charset );
free( psz_charset );
if( p_sap->iconvHandle == (vlc_iconv_t)(-1) )
msg_Warn( p_sap, "Unable to do requested conversion" );
p_sap->pf_add = announce_SAPAnnounceAdd;