python カンマ区切りの文字列を，别々の集にしたい时のサンプル

Posted 2021-05-09

# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import argparse
import apache_beam as beam
from apache_beam import DoFn
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.typehints import with_input_types, with_output_types


@with_input_types(str)
@with_output_types(str)
class Split(DoFn):
    def __init__(self, sep):
        super(ToDuplicate, self).__init__()
        self.sep = sep

    def process(self, element, *args, **kwargs):
        for elem in element.split(self.sep):
            yield elem


def run(argv=None):
    """
    カンマ区切りの文字列を、別々のCollectionにしたい時のサンプル
    """
    parser = argparse.ArgumentParser()
    _, pipeline_args = parser.parse_known_args(argv)
    pipeline = beam.Pipeline(options=PipelineOptions())

    (pipeline
     | beam.Create(["Hoge,Fuga,Piyo"])
     | beam.ParDo(Split(sep=","))
     | beam.Map(lambda row: print(row))
     )
     
    # 出力結果
    _ = """
    Hoge
    Fuga
    Piyo
    """

    result = pipeline.run()
    result.wait_until_finish()


if __name__ == '__main__':
    run()