将文本按行标识符进行分割

Posted 2021-02-16 wenlin-gk

def split_docs(lines, separator):
    """
    :note: The English sentence is in the front,
        the Chinese sentence is in the back,
        and the two are separated by a separator.
    """
    if not lines:
        return [], []

    eng_lines = []
    chn_lines = []

    chn_begin_condition = False
    for line in lines:
        if chn_begin_condition:
            chn_lines.append(line)
        else:
            chn_begin_condition = line.lstrip().startswith(separator)
            if not chn_begin_condition:
                eng_lines.append(line)

    return eng_lines, chn_lines