C++如何使用C++读取Unicode字符(例如,印地语脚本),或者通过其他编程语言有更好的方法吗?

separators = [u"।", u",", u"."]
text = open("indiantext.txt").read()
#This converts the encoded text to an internal unicode object, where
# all characters are properly recognized as an entity:
text = text.decode("utf-8")

#this breaks the text on the white spaces, yielding a list of words:
words = text.split()

counter = 1

output = ""
for word in words:
    #if the last char is a separator, and is joined to the word:
    if word[-1] in separators and len(word) > 1:
        #word up to the second to last char:
        output += word[:-1] + u"(%d) " % counter
        counter += 1
        #last char
        output += word[-1] +  u"(%d) " % counter
    else:
        output += word + u"(%d) " % counter
    counter += 1

print output

http://python.org

use utf8; use strict; use warnings;
use Encode qw(decode);
my $index;
join ' ', map { $index++; "$_($index)" } split /\s+|(?=।)/, decode 'UTF-8', <>;
# returns भारत(1) का(2) इतिहास(3) काफी(4) समदध(5) एव(6) विसतत(7) ह(8) ।(9)

STDIN

utfcpp

#!/usr/bin/env python
# encoding: utf-8

string = u"भारत का इतिहास काफी समृद्ध एवं विस्तृत है।"
parts = []
for part in string.split():
    parts.extend(part.split(u"।"))
print "No of Parts: %d" % len(parts)
print "Parts: %s" % parts

No of Parts: 9
Parts: [u'\u092d\u093e\u0930\u0924', u'\u0915\u093e', u'\u0907\u0924\u093f\u0939\u093e\u0938', u'\u0915\u093e\u092b\u0940', u'\u0938\u092e\u0943\u0926\u094d\u0927', u'\u090f\u0935\u0902', u'\u0935\u093f\u0938\u094d\u0924\u0943\u0924', u'\u0939\u0948', u'']

NLTK

ICU - International Components for Unicode

http://site.icu-project.org/

std::wstring wchar_t L' ' input.find_first_of(L" ।")

FILE * fp = _wfopen( L"fname",L"r" );
wchar_t buf[1000];
while( fgetws(buf,999, fp ) )   {
    fwprintf(L"%s",buf);
}

// convert UTF-8 to UNICODE

    void String2WString( std::wstring& ws, const std::string& s )
    {
        ws.clear();
        int nLenOfWideCharStr = MultiByteToWideChar(CP_ACP, 0, 
            s.c_str(), s.length(), NULL, 0); 
        PWSTR pWideCharStr = (PWSTR)HeapAlloc(GetProcessHeap(), 0, 
            nLenOfWideCharStr * sizeof(wchar_t)+2); 
        if (pWideCharStr == NULL)         
            return; 
        MultiByteToWideChar(CP_ACP, 0, 
            s.c_str(), s.length(), 
            pWideCharStr, nLenOfWideCharStr);
        *(pWideCharStr+nLenOfWideCharStr ) = L'\0';
        ws = pWideCharStr ;
        HeapFree(GetProcessHeap(), 0, pWideCharStr); 

    }

    // read UTF-8
FILE * fp = fopen( "fname","r" );
char buf[1000];
std::string aline;
std::wstring wline;
std::vector< std::wstring> vline;
while( fgets(buf,999, fp ) )    {
    aline = buf;
    String2WString( wline, aline );
    vline.push_back( wline );
}