#!/usr/bin/python2
# coding: utf-8
import codecs
from lxml import etree
from copy import deepcopy
from random import choice
FILE_SIZE = 1 * 1000 * 1000 # 1MB - размер одного файла
TOTAL_SIZE = 14.5 * 1000 * 1000 # 10MB - размер всех файлов
FILENAME = "alut-txt-all.flextext"
letters = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-'
letters += u'абсгдеёжзийклмнопрстуфхцчшщъыьЭЮЯАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'
def morph_word(word):
l = []
for i in xrange(len(word)):
if word[i] in letters:
l.append(i)
if 0 in l:
l.remove(0)
if l:
for k in xrange(3):
i = choice(l)
j = choice(l)
if i > j: i, j = j, i
if i != j:
word = word[:i] + word[j] + word[i+1:j] + word[i] + word[j+1:]
return word
def morph_sentence(text):
new_text = ''
for word in text.split(' '):
new_text += morph_word(word) + ' '
return new_text.strip()
def tag_size(tag):
return len(etree.tostring(tag, encoding='unicode'))
f = codecs.open(FILENAME, 'r', 'utf-8')
tree = etree.parse(f)
f.close()
document = tree.getroot()
new_document = deepcopy(document)
current_document = etree.Element('document', version="2")
cur_total_size = 0
cur_number = 1
# копируем текст без изменений
for interlinear_text in document.findall('interlinear-text'):
if tag_size(current_document) + tag_size(interlinear_text) > FILE_SIZE:
f = codecs.open('new_%03i_%s'%(cur_number,FILENAME), 'w', 'utf-8')
f.write(etree.tostring(current_document, encoding='unicode'))
f.close()
cur_total_size += FILE_SIZE
cur_number += 1
current_document = etree.Element('document', version="2")
if cur_total_size + tag_size(current_document) + tag_size(interlinear_text) > TOTAL_SIZE:
break
new_interlinear_text = etree.SubElement(current_document, 'interlinear-text', guid=interlinear_text.get('guid'))
for ch in interlinear_text.getchildren():
new_interlinear_text.append( deepcopy(ch) )
# множим текст
flag = True
while flag:
for interlinear_text in document.findall('interlinear-text'):
if tag_size(current_document) + tag_size(interlinear_text) > FILE_SIZE:
f = codecs.open('new_%03i_%s'%(cur_number,FILENAME), 'w', 'utf-8')
f.write(etree.tostring(current_document, encoding='unicode'))
f.close()
cur_total_size += tag_size(current_document)
cur_number += 1
current_document = etree.Element('document', version="2")
if cur_total_size + tag_size(current_document) + tag_size(interlinear_text) > TOTAL_SIZE:
flag = False
break
# - создаём пустой <interlinear-text/>
new_interlinear_text = etree.SubElement(current_document, 'interlinear-text', guid=interlinear_text.get('guid'))
# - копируем в него всё из interlinear_text
for ch in interlinear_text.getchildren():
new_interlinear_text.append( deepcopy(ch) )
for title in new_interlinear_text.findall('item'):
if title.get('type') == 'title':
title.text = morph_sentence(title.text)
# изменяем
paragraphs = new_interlinear_text.find('paragraphs')
for paragraph in paragraphs.findall('paragraph'):
phrases = paragraph.find('phrases')
if phrases is None: continue
for phrase in phrases.findall('phrase'):
words = phrase.find('words')
for word in words:
item_txt = word.find('item')
morphemes = word.find('morphemes')
if morphemes is None: continue
for morph in morphemes.findall('morph'):
if morph.get('type') == 'stem':
for item in morph.findall('item'):
if item.text:
item.text = morph_sentence(item.text)
if item.get('lang') == 'alr-x-lat':
item_txt.text = item.text
for item in phrase.findall('item'):
if (item.get('type') == 'gls') and item.text:
item.text = morph_sentence(item.text)
# записываем "остатки" в новый файл
if current_document.find('interlinear-text') is not None:
f = codecs.open('new_%03i_%s'%(cur_number,FILENAME), 'w', 'utf-8')
f.write(etree.tostring(current_document, encoding='unicode'))
f.close()