tested django-newsletter

This commit is contained in:
Esther Kleinhenz
2018-10-26 15:21:17 +02:00
parent 7ed667d043
commit 811b7c5453
2352 changed files with 448169 additions and 1 deletions
@@ -0,0 +1,281 @@
#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# Written by Martin v. Loewis <loewis@informatik.hu-berlin.de>
#
# Changed by Christian 'Tiran' Heimes <tiran@cheimes.de> for the placeless
# translation service (PTS) of Zope
#
# Fixed some bugs and updated to support msgctxt
# by Hanno Schlichting <hanno@hannosch.eu>
"""Generate binary message catalog from textual translation description.
This program converts a textual Uniforum-style message catalog (.po file) into
a binary GNU catalog (.mo file). This is essentially the same function as the
GNU msgfmt program, however, it is a simpler implementation.
This file was taken from Python-2.3.2/Tools/i18n and altered in several ways.
Now you can simply use it from another python module:
from msgfmt import Msgfmt
mo = Msgfmt(po).get()
where po is path to a po file as string, an opened po file ready for reading or
a list of strings (readlines of a po file) and mo is the compiled mo file as
binary string.
Exceptions:
* IOError if the file couldn't be read
* msgfmt.PoSyntaxError if the po file has syntax errors
"""
import array
from ast import literal_eval
import codecs
from email.parser import HeaderParser
import struct
import sys
PY3 = sys.version_info[0] == 3
if PY3:
def header_charset(s):
p = HeaderParser()
return p.parsestr(s).get_content_charset()
import io
BytesIO = io.BytesIO
FILE_TYPE = io.IOBase
else:
def header_charset(s):
p = HeaderParser()
return p.parsestr(s.encode('utf-8', 'ignore')).get_content_charset()
from cStringIO import StringIO as BytesIO
FILE_TYPE = file
class PoSyntaxError(Exception):
""" Syntax error in a po file """
def __init__(self, msg):
self.msg = msg
def __str__(self):
return 'Po file syntax error: %s' % self.msg
class Msgfmt:
def __init__(self, po, name='unknown'):
self.po = po
self.name = name
self.messages = {}
self.openfile = False
# Start off assuming latin-1, so everything decodes without failure,
# until we know the exact encoding
self.encoding = 'latin-1'
def readPoData(self):
""" read po data from self.po and return an iterator """
output = []
if isinstance(self.po, str):
output = open(self.po, 'rb')
elif isinstance(self.po, FILE_TYPE):
self.po.seek(0)
self.openfile = True
output = self.po
elif isinstance(self.po, list):
output = self.po
if not output:
raise ValueError("self.po is invalid! %s" % type(self.po))
if isinstance(output, FILE_TYPE):
# remove BOM from the start of the parsed input
first = output.readline()
if len(first) == 0:
return output.readlines()
if first.startswith(codecs.BOM_UTF8):
first = first.lstrip(codecs.BOM_UTF8)
return [first] + output.readlines()
return output
def add(self, context, id, string, fuzzy):
"Add a non-empty and non-fuzzy translation to the dictionary."
if string and not fuzzy:
# The context is put before the id and separated by a EOT char.
if context:
id = context + u'\x04' + id
if not id:
# See whether there is an encoding declaration
charset = header_charset(string)
if charset:
# decode header in proper encoding
string = string.encode(self.encoding).decode(charset)
if not PY3:
# undo damage done by literal_eval in Python 2.x
string = string.encode(self.encoding).decode(charset)
self.encoding = charset
self.messages[id] = string
def generate(self):
"Return the generated output."
# the keys are sorted in the .mo file
keys = sorted(self.messages.keys())
offsets = []
ids = strs = b''
for id in keys:
msg = self.messages[id].encode(self.encoding)
id = id.encode(self.encoding)
# For each string, we need size and file offset. Each string is
# NUL terminated; the NUL does not count into the size.
offsets.append((len(ids), len(id), len(strs),
len(msg)))
ids += id + b'\0'
strs += msg + b'\0'
output = b''
# The header is 7 32-bit unsigned integers. We don't use hash tables,
# so the keys start right after the index tables.
keystart = 7 * 4 + 16 * len(keys)
# and the values start after the keys
valuestart = keystart + len(ids)
koffsets = []
voffsets = []
# The string table first has the list of keys, then the list of values.
# Each entry has first the size of the string, then the file offset.
for o1, l1, o2, l2 in offsets:
koffsets += [l1, o1 + keystart]
voffsets += [l2, o2 + valuestart]
offsets = koffsets + voffsets
# Even though we don't use a hashtable, we still set its offset to be
# binary compatible with the gnu gettext format produced by:
# msgfmt file.po --no-hash
output = struct.pack("Iiiiiii",
0x950412de, # Magic
0, # Version
len(keys), # # of entries
7 * 4, # start of key index
7 * 4 + len(keys) * 8, # start of value index
0, keystart) # size and offset of hash table
if PY3:
output += array.array("i", offsets).tobytes()
else:
output += array.array("i", offsets).tostring()
output += ids
output += strs
return output
def get(self):
""" """
self.read()
# Compute output
return self.generate()
def read(self, header_only=False):
""" """
ID = 1
STR = 2
CTXT = 3
section = None
fuzzy = 0
msgid = msgstr = msgctxt = u''
# Parse the catalog
lno = 0
for l in self.readPoData():
l = l.decode(self.encoding)
lno += 1
# If we get a comment line after a msgstr or a line starting with
# msgid or msgctxt, this is a new entry
if section == STR and (l[0] == '#' or (l[0] == 'm' and
(l.startswith('msgctxt') or l.startswith('msgid')))):
self.add(msgctxt, msgid, msgstr, fuzzy)
section = None
fuzzy = 0
# If we only want the header we stop after the first message
if header_only:
break
# Record a fuzzy mark
if l[:2] == '#,' and 'fuzzy' in l:
fuzzy = 1
# Skip comments
if l[0] == '#':
continue
# Now we are in a msgctxt section
if l.startswith('msgctxt'):
section = CTXT
l = l[7:]
msgctxt = u''
# Now we are in a msgid section, output previous section
elif (l.startswith('msgid') and
not l.startswith('msgid_plural')):
if section == STR:
self.add(msgid, msgstr, fuzzy)
section = ID
l = l[5:]
msgid = msgstr = u''
is_plural = False
# This is a message with plural forms
elif l.startswith('msgid_plural'):
if section != ID:
raise PoSyntaxError(
'msgid_plural not preceeded by '
'msgid on line %d of po file %s' %
(lno, repr(self.name)))
l = l[12:]
msgid += u'\0' # separator of singular and plural
is_plural = True
# Now we are in a msgstr section
elif l.startswith('msgstr'):
section = STR
if l.startswith('msgstr['):
if not is_plural:
raise PoSyntaxError(
'plural without msgid_plural '
'on line %d of po file %s' %
(lno, repr(self.name)))
l = l.split(']', 1)[1]
if msgstr:
# Separator of the various plural forms
msgstr += u'\0'
else:
if is_plural:
raise PoSyntaxError(
'indexed msgstr required for '
'plural on line %d of po file %s' %
(lno, repr(self.name)))
l = l[6:]
# Skip empty lines
l = l.strip()
if not l:
continue
# TODO: Does this always follow Python escape semantics?
try:
l = literal_eval(l)
except Exception as msg:
raise PoSyntaxError(
'%s (line %d of po file %s): \n%s' %
(msg, lno, repr(self.name), l))
if isinstance(l, bytes):
l = l.decode(self.encoding)
if section == CTXT:
msgctxt += l
elif section == ID:
msgid += l
elif section == STR:
msgstr += l
else:
raise PoSyntaxError(
'error on line %d of po file %s' %
(lno, repr(self.name)))
# Add last entry
if section == STR:
self.add(msgctxt, msgid, msgstr, fuzzy)
if self.openfile:
self.po.close()
def getAsFile(self):
return BytesIO(self.get())
@@ -0,0 +1,41 @@
# Some comments
# Some more comments
msgid ""
msgstr ""
"Project-Id-Version: test\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
# comment1
#. More comments: "default1"
#: folder1/file1
#: folder2/file1
msgid "msgid1"
msgstr "msgstr1"
# comment2
#: file2
msgid "msgid2"
msgstr "msgstr2"
# Default: "default3"
#: file3
msgid "msgid3"
msgstr "msgstr3"
#. Default: "default4"
msgid "msgid4"
msgstr "msgstr4"
# comment5
msgid "msgid5"
msgstr "msgstr5"
#, fuzzy
msgid "msgid6"
msgstr "msgstr6"
@@ -0,0 +1,41 @@
# Some comments
# Some more comments
msgid ""
msgstr ""
"Project-Id-Version: test\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Hanno Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
# comment1
#. More comments: "default1"
#: folder1/file1
#: folder2/file1
msgid "msgid1"
msgstr ""
# comment2
#: file2
msgid "msgid2"
msgstr ""
# Default: "default3"
#: file3
msgid "msgid3"
msgstr ""
#. Default: "default4"
msgid "msgid4"
msgstr ""
# comment5
msgid "msgid5"
msgstr ""
#, fuzzy
msgid "msgid6"
msgstr ""
@@ -0,0 +1,21 @@
# Some comments
msgid ""
msgstr ""
"Project-Id-Version: test2\n"
"POT-Creation-Date: 2007-05-31 22:15+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
# comment1
#. More comments: "default1"
#: folder1/file1
msgid "msgid1"
msgstr "msgstr1"
#, fuzzy
msgid "msgid2"
msgstr "msgstr2"
@@ -0,0 +1,36 @@
msgid ""
msgstr ""
"Project-Id-Version: test3\n"
"POT-Creation-Date: 2007-05-31 22:15+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
# comment1
#. More comments: "default1"
#: folder1/file1
msgid "msgid1"
msgstr "msgstr1"
#, fuzzy
msgid "msgid2"
msgstr "msgstr2"
msgctxt "msgctext3"
msgid "msgid3"
msgstr "msgstr3"
# comment4
#. More comments: "default4"
#: folder4/file4
msgctxt "msgctext4"
msgid "msgid4"
msgstr "msgstr4"
#, fuzzy
msgctxt "msgctext5"
msgid "msgid5"
msgstr "msgstr5"
@@ -0,0 +1,22 @@
# Translation of foo.pot to Afrikaans
# Mighty translator, 2007
msgid ""
msgstr ""
"Project-Id-Version: foo\n"
"POT-Creation-Date: 2007-09-01 12:34+0000\n"
"PO-Revision-Date: 2006-05-26 14:12+0200\n"
"Last-Translator: Mighty translator\n"
"Language-Team: Afrikaans <nomail@mail.no>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=iso-8859-1\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0\n"
"Language-Code: af\n"
"Language-Name: Afrikaans\n"
"Preferred-Encodings: utf-8 latin1\n"
"Domain: foo\n"
#. Default: "Message 1"
#: ./foo/bar.py:42
msgid "message_1"
msgstr "Message 1"
@@ -0,0 +1,17 @@
msgid ""
msgstr ""
"Project-Id-Version: test\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
msgid "msgid1"
msgstr "føø"
msgid "msgid2"
msgstr "føø
bår"
@@ -0,0 +1,14 @@
msgid ""
msgstr ""
"Project-Id-Version: Tøst 1.0\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"PO-Revision-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Föø Bår <foo@bar.com>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
msgid "msgid1"
msgstr "føø"
@@ -0,0 +1,109 @@
# -*- coding: utf-8 -*-
import os
from pythongettext.msgfmt import Msgfmt
from pythongettext.msgfmt import PoSyntaxError
try:
import unittest2 as unittest
except ImportError: # Python 2.7 or newer
import unittest
FOLDER = os.path.dirname(__file__)
class TestWriter(unittest.TestCase):
def compare_po_mo(self, poname, moname):
po_file = None
mo_file = None
try:
po_file = open(os.path.join(FOLDER, poname), 'rb')
po = Msgfmt(po_file).get()
mo_file = open(os.path.join(FOLDER, moname), 'rb')
mo = b''.join(mo_file.readlines())
finally:
if po_file is not None:
po_file.close()
if mo_file is not None:
mo_file.close()
self.assertEqual(mo, po)
def test_empty(self):
self.compare_po_mo('test_empty.po', 'test_empty.mo')
def test_test(self):
self.compare_po_mo('test.po', 'test.mo')
def test_test2(self):
self.compare_po_mo('test2.po', 'test2.mo')
def test_msgctxt(self):
self.compare_po_mo('test3.po', 'test3.mo')
def test_test4(self):
po_file = open(os.path.join(FOLDER, 'test4.po'), 'rb')
po = Msgfmt(po_file)
po.read(header_only=True)
po_file.close()
self.assertTrue(
po.messages[u''].startswith('Project-Id-Version: foo'))
self.assertEqual(po.encoding, u'iso-8859-1')
def test_test5(self):
po_file = open(os.path.join(FOLDER, 'test5.po'), 'rb')
po = Msgfmt(po_file)
try:
with self.assertRaises(PoSyntaxError):
po.read()
finally:
po_file.close()
self.assertEqual(po.encoding, u'utf-8')
def test_test5_unicode_name(self):
po_file = open(os.path.join(FOLDER, 'test5.po'), 'rb')
po = Msgfmt(po_file, name=u'dømain')
try:
with self.assertRaises(PoSyntaxError):
po.read()
finally:
po_file.close()
self.assertEqual(po.encoding, u'utf-8')
def test_test6(self):
self.compare_po_mo('test6.po', 'test6.mo')
def test_test6_unicode_header(self):
po_file = open(os.path.join(FOLDER, 'test6.po'), 'rb')
po = Msgfmt(po_file)
po.read(header_only=True)
po_file.close()
self.assertTrue(po.messages[u''].startswith(
u'Project-Id-Version: Tøst 1.0'))
self.assertEqual(po.encoding, u'utf-8')
def test_escape(self):
po_file = open(os.path.join(FOLDER, 'test_escape.po'), 'rb')
po = Msgfmt(po_file)
try:
with self.assertRaises(PoSyntaxError) as e:
po.read()
self.assertTrue('line 19' in e.exception.msg)
self.assertEqual(po.encoding, u'utf-8')
finally:
po_file.close()
def test_unicode_bom(self):
self.compare_po_mo('test_unicode_bom.po', 'test_unicode_bom.mo')
def test_plural(self):
po_file = open(os.path.join(FOLDER, 'test_plural.po'), 'rb')
po = Msgfmt(po_file)
try:
po.read()
finally:
po_file.close()
self.assertEqual(
set(po.messages.keys()),
set([u'', u'm1', u'm2 ø\x00{d} ømsgid',
u'øcontext\x04m3 ø\x00{d} ømsgid context']))
@@ -0,0 +1,19 @@
msgid ""
msgstr ""
"Project-Id-Version: test\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
msgid "msgid1"
msgstr "Hello ${foo}"
msgid "msgid2"
msgstr "Hellø \"bar\""
msgid "msgid3"
msgstr "Hellø "bar\""
@@ -0,0 +1,26 @@
msgid ""
msgstr ""
"Project-Id-Version: test_plural\n"
"POT-Creation-Date: 2016-01-04 16:20+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;\n"
msgid "m1"
msgstr "msgstr"
msgid "m2 ø"
msgid_plural "{d} ømsgid"
msgstr[0] "ø"
msgstr[1] "øø"
msgstr[2] "øøø"
msgctxt "øcontext"
msgid "m3 ø"
msgid_plural "{d} ømsgid context"
msgstr[0] "ø"
msgstr[1] "øø"
msgstr[2] "øøø"
@@ -0,0 +1,13 @@
msgid ""
msgstr ""
"Project-Id-Version: test\n"
"POT-Creation-Date: 2007-05-31 19:30+0100\n"
"Last-Translator: Hanno C. Schlichting <hanno@hannosch.eu>\n"
"Language-Team: <>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=1; plural=0;\n"
msgid "msgid1"
msgstr "føø"