diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e4424bd6..386712955 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -206,6 +206,7 @@ add_subdirectory(external/groff) add_subdirectory(external/nano) add_subdirectory(external/man) add_subdirectory(external/bc) +add_subdirectory(external/vim) add_subdirectory(external/zip/zip/zip30) add_subdirectory(external/zip/unzip/unzip-5.52) add_subdirectory(tools) diff --git a/src/CoreServices/CMakeLists.txt b/src/CoreServices/CMakeLists.txt index 0a45a5673..b25510331 100644 --- a/src/CoreServices/CMakeLists.txt +++ b/src/CoreServices/CMakeLists.txt @@ -40,7 +40,8 @@ set(CoreServices_SRCS OpenTransport.cpp Timer.cpp CoreServicesPriv.cpp - #../external/libcxx/src/dso_handle.c + TextCommon.cpp + TextEncodingConverter.cpp ) if (WITH_COREAUDIO) diff --git a/src/CoreServices/CoreServices.h b/src/CoreServices/CoreServices.h index 10d8eb688..357581b40 100644 --- a/src/CoreServices/CoreServices.h +++ b/src/CoreServices/CoreServices.h @@ -23,6 +23,8 @@ #include #include #include +#include +#include #endif diff --git a/src/CoreServices/TextCommon.cpp b/src/CoreServices/TextCommon.cpp new file mode 100644 index 000000000..c7cedf125 --- /dev/null +++ b/src/CoreServices/TextCommon.cpp @@ -0,0 +1,31 @@ +/* +This file is part of Darling. + +Copyright (C) 2017 Lubos Dolezel + +Darling is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Darling is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Darling. If not, see . +*/ + +#include "TextCommon.h" + +TextEncoding CreateTextEncoding(TextEncodingBase encodingBase, TextEncodingVariant encodingVariant, TextEncodingFormat encodingFormat) +{ + TextEncoding rv = encodingBase & 0xff; + rv |= (encodingVariant << 8) & 0xff00; + rv |= (encodingFormat << 16) & 0xff0000; + return rv; +} + + + diff --git a/src/CoreServices/TextCommon.h b/src/CoreServices/TextCommon.h new file mode 100644 index 000000000..bf34b3fca --- /dev/null +++ b/src/CoreServices/TextCommon.h @@ -0,0 +1,75 @@ +/* +This file is part of Darling. + +Copyright (C) 2017 Lubos Dolezel + +Darling is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Darling is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Darling. If not, see . +*/ + +#ifndef _TEXTCOMMON_H +#define _TEXTCOMMON_H +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef UInt8* TextPtr; +typedef const UInt8* ConstTextPtr; + +enum { + kTextEncodingUnicodeDefault = 0x100 +}; + +enum { + kTextEncodingDefaultVariant = 0 +}; + +enum { + kTextEncodingDefaultFormat = 0, + kUnicodeUTF16Format = 0, + kUnicodeUTF7Format = 1, + kUnicodeUTF8Format = 2, + kUnicodeUTF32Format = 3, + kUnicodeUTF16BEFormat = 4, + kUnicodeUTF16LEFormat = 5, + kUnicodeUTF32BEFormat = 6, + kUnicodeUTF32LEFormat = 7, + kUnicodeSCSUFormat = 8, + kUnicode16BitFormat = 0, + kUnicode32BitFormat = 3, +}; + +enum { + kUnicodeNoSubset = 0, + kUnicodeNormalizationFormD = 5, + kUnicodeNormalizationFormC = 3, + kUnicodeCanonicalCompVariant = kUnicodeNormalizationFormC, + kUnicodeHFSPlusDecompVariant = 8, + kUnicodeHFSPlusCompVariant = 9, +}; + +typedef UInt32 TextEncoding; +typedef UInt32 TextEncodingBase; +typedef UInt32 TextEncodingFormat; +typedef UInt32 TextEncodingVariant; + +TextEncoding CreateTextEncoding(TextEncodingBase encodingBase, TextEncodingVariant encodingVariant, TextEncodingFormat encodingFormat); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/CoreServices/TextEncodingConverter.cpp b/src/CoreServices/TextEncodingConverter.cpp new file mode 100644 index 000000000..3c1a834a9 --- /dev/null +++ b/src/CoreServices/TextEncodingConverter.cpp @@ -0,0 +1,233 @@ +/* +This file is part of Darling. + +Copyright (C) 2017 Lubos Dolezel + +Darling is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Darling is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Darling. If not, see . +*/ + +#include "TextEncodingConverter.h" +#include +#include +#include "MacErrors.h" + +struct OpaqueTECObjectRef +{ + UConverter* inputConverter; + UConverter* outputConverter; + const UNormalizer2* normalizer; + UChar buffer[64]; + size_t bufferUsed; +}; + +static UConverter* createConverter(TextEncodingBase base, TextEncodingFormat format) +{ + switch (base) + { + case kTextEncodingUnicodeDefault: + { + const char* enc; + UErrorCode error = U_ZERO_ERROR; + + switch (format) + { + case kUnicodeUTF16Format: + enc = "UTF-16"; + break; + case kUnicodeUTF7Format: + enc = "UTF-7"; + break; + case kUnicodeUTF8Format: + enc = "UTF-8"; + break; + case kUnicodeUTF32Format: + enc = "UTF-32"; + break; + case kUnicodeUTF16BEFormat: + enc = "UTF-16BE"; + break; + case kUnicodeUTF16LEFormat: + enc = "UTF-16LE"; + break; + case kUnicodeUTF32BEFormat: + enc = "UTF-32BE"; + break; + case kUnicodeUTF32LEFormat: + enc = "UTF-32LE"; + break; + default: + return NULL; + } + + return ucnv_open(enc, &error); + } + default: + return NULL; + } +} + +OSStatus TECCreateConverter(TECObjectRef *newEncodingConverter, TextEncoding inputEncoding, TextEncoding outputEncoding) +{ + TextEncodingFormat format; + TextEncodingBase base; + TextEncodingVariant variant; + OpaqueTECObjectRef* obj = new OpaqueTECObjectRef; + + obj->inputConverter = obj->outputConverter = NULL; + obj->normalizer = NULL; + obj->bufferUsed = 0; + + format = (inputEncoding >> 16) & 0xff; + base = (inputEncoding >> 0) & 0xff; + variant = (inputEncoding >> 8) & 0xff; + + obj->inputConverter = createConverter(base, format); + if (!obj->inputConverter) + { + TECDisposeConverter(obj); + *newEncodingConverter = NULL; + return unimpErr; + } + + format = (outputEncoding >> 16) & 0xff; + base = (outputEncoding >> 0) & 0xff; + variant = (outputEncoding >> 8) & 0xff; + + obj->outputConverter = createConverter(base, format); + if (!obj->outputConverter) + { + TECDisposeConverter(obj); + *newEncodingConverter = NULL; + return unimpErr; + } + + if (base == kTextEncodingUnicodeDefault) + { + switch (variant) + { + case kUnicodeNoSubset: + break; + case kUnicodeNormalizationFormD: + { + UErrorCode error = U_ZERO_ERROR; + obj->normalizer = unorm2_getNFDInstance(&error); + break; + } + case kUnicodeNormalizationFormC: + { + UErrorCode error = U_ZERO_ERROR; + obj->normalizer = unorm2_getNFCInstance(&error); + break; + } + case kUnicodeHFSPlusDecompVariant: + case kUnicodeHFSPlusCompVariant: + { + UErrorCode error = U_ZERO_ERROR; + obj->normalizer = unorm2_getNFKDInstance(&error); + break; + } + } + } + + *newEncodingConverter = obj; + return noErr; +} + +OSStatus TECConvertText(TECObjectRef encodingConverter, ConstTextPtr inputBuffer, + ByteCount inputBufferLength, ByteCount *actualInputLength, + TextPtr outputBuffer, ByteCount outputBufferLength, ByteCount *actualOutputLength) +{ + if (actualInputLength != NULL) + *actualInputLength = 0; + *actualOutputLength = 0; + + while (outputBufferLength > 0) + { + if (encodingConverter->bufferUsed > 0) + { + // flush buffer + UErrorCode error = U_ZERO_ERROR; + char* target = (char*) outputBuffer; + const UChar* source = encodingConverter->buffer; + ByteCount inputUsed; + + + ucnv_fromUnicode(encodingConverter->outputConverter, + &target, target + outputBufferLength, + &source, source + encodingConverter->bufferUsed, + NULL, false, &error); + + if (error != U_ZERO_ERROR && error != U_BUFFER_OVERFLOW_ERROR) + return paramErr; + + *actualOutputLength = target - ((char*)outputBuffer); + + inputUsed = source - encodingConverter->buffer; + if (inputUsed < encodingConverter->bufferUsed) + { + memmove(encodingConverter->buffer, + encodingConverter->buffer + inputUsed, + encodingConverter->bufferUsed - inputUsed); + } + encodingConverter->bufferUsed -= inputUsed; + outputBuffer += *actualOutputLength; + outputBufferLength -= *actualOutputLength; + + if (error == U_BUFFER_OVERFLOW_ERROR) + break; + } + + if (inputBufferLength <= 0) + break; + + // Consume input + { + UChar* target = encodingConverter->buffer + encodingConverter->bufferUsed; + const char* source = (const char*) inputBuffer; + UErrorCode error = U_ZERO_ERROR; + + ucnv_toUnicode(encodingConverter->inputConverter, + &target, encodingConverter->buffer + (sizeof(encodingConverter->buffer) / sizeof(encodingConverter->buffer[0])), + &source, source + inputBufferLength, + NULL, false, &error); + + if (error != U_ZERO_ERROR && error != U_BUFFER_OVERFLOW_ERROR) + return paramErr; + + encodingConverter->bufferUsed += target - (encodingConverter->buffer + encodingConverter->bufferUsed); + *actualInputLength += source - ((const char*)inputBuffer); + inputBufferLength -= source - ((const char*)inputBuffer); + inputBuffer = (ConstTextPtr) source; + } + + // TODO: normalize + // Normalization may cause the data to no longer fit into our internal buffer :-/ + } + + return noErr; +} + +OSStatus TECFlushText(TECObjectRef encodingConverter, TextPtr outputBuffer, ByteCount outputBufferLength, ByteCount *actualOutputLength) +{ + return TECConvertText(encodingConverter, NULL, 0, NULL, outputBuffer, outputBufferLength, actualOutputLength); +} + +OSStatus TECDisposeConverter(TECObjectRef conv) +{ + ucnv_close(conv->inputConverter); + ucnv_close(conv->outputConverter); + delete conv; + return noErr; +} + diff --git a/src/CoreServices/TextEncodingConverter.h b/src/CoreServices/TextEncodingConverter.h new file mode 100644 index 000000000..63a7d076b --- /dev/null +++ b/src/CoreServices/TextEncodingConverter.h @@ -0,0 +1,44 @@ +/* +This file is part of Darling. + +Copyright (C) 2017 Lubos Dolezel + +Darling is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Darling is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Darling. If not, see . +*/ + +#ifndef _TEXTENCODINGCONVERTER_H +#define _TEXTENCODINGCONVERTER_H +#include +#include "TextCommon.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct OpaqueTECObjectRef* TECObjectRef; + +OSStatus TECCreateConverter(TECObjectRef *newEncodingConverter, TextEncoding inputEncoding, TextEncoding outputEncoding); + +OSStatus TECConvertText(TECObjectRef encodingConverter, ConstTextPtr inputBuffer, ByteCount inputBufferLength, ByteCount *actualInputLength, TextPtr outputBuffer, ByteCount outputBufferLength, ByteCount *actualOutputLength); + +OSStatus TECFlushText(TECObjectRef encodingConverter, TextPtr outputBuffer, ByteCount outputBufferLength, ByteCount *actualOutputLength); + +OSStatus TECDisposeConverter(TECObjectRef newEncodingConverter); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/external/vim b/src/external/vim index 1de90e101..195f439d3 160000 --- a/src/external/vim +++ b/src/external/vim @@ -1 +1 @@ -Subproject commit 1de90e101b264be43a91a4e0c09b5570c0b6a6c7 +Subproject commit 195f439d3c5711d38f5ede5aac3719543febec1e