{"allowedRenditionsWidth":["320","480","640","768","960","1200","1440","1920"],"templateName":"blog-page","cssClassNames":"blog-page page basicpage summit-page","canonicalLink":"https://www.snowflake.com/en/blog/snowpipe-streaming-iceberg-tables/","robotsTags":["index","follow"],"language":"en","description":"Discover how to easily stream data into Snowflake-managed Apache Iceberg tables using Snowpipe Streaming.","title":"Streaming Data into Apache Iceberg with Snowflake","analyticsPageType":"homepage","analyticsCategory":"general","analyticsSubCategory":"","excludeFromAnalytics":false,"analyticsEnabled":true,":hierarchyType":"page",":path":"/content/snowflake-site/global/en/blog/snowpipe-streaming-iceberg-tables",":mappedPath":"/en/blog/snowpipe-streaming-iceberg-tables/",":type":"snowflake-site/components/structure/page","analyticsDebugMode":false,"analyticsData":{"excludeFromAnalytics":false,"subCategory":"","pageType":"homepage","templateName":"blog-page","siteName":"snowflake","pageUrl":"/content/snowflake-site/global/en/blog/snowpipe-streaming-iceberg-tables","language":"en","category":"general","pageName":"Stream to Apache Iceberg Easily with Snowpipe Streaming","contentTags":["snowflake-site:taxonomy/blog/product-and-technology/data-engineering","snowflake-site:taxonomy/product/data-engineering","snowflake-site:taxonomy/snowflake-feature/apache-iceberg","snowflake-site:taxonomy/snowflake-feature/ingestion"]},"isPasswordProtected":false,"analyticsContentTags":["snowflake-site:taxonomy/blog/product-and-technology/data-engineering","snowflake-site:taxonomy/product/data-engineering","snowflake-site:taxonomy/snowflake-feature/apache-iceberg","snowflake-site:taxonomy/snowflake-feature/ingestion"],"locale":"en",":items":{"root":{"columnClassNames":{"experiencefragment-banner":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-sub-header":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-pre-footer":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-header":"aem-GridColumn aem-GridColumn--default--12","markup_editor-table":"aem-GridColumn aem-GridColumn--default--12","responsivegrid":"aem-GridColumn aem-GridColumn--default--12","experiencefragment-footer":"aem-GridColumn aem-GridColumn--default--12","markup_editor":"aem-GridColumn aem-GridColumn--default--12","container_47873732":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnCount":12,":items":{"experiencefragment-banner":{"id":"experiencefragment-ccec57afbd","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/pushdown-banner/pushdown-banner-blank/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/pushdown-banner/pushdown-banner-blank.xfmodel.json"},"experiencefragment-header":{"id":"experiencefragment-0bcc5cd333","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/mega-nav-header/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/mega-nav-header/master.xfmodel.json","languageNavPath":"/content/snowflake-site/global/en/blog/snowpipe-streaming-iceberg-tables.languagenav.json","appliedCssClassNames":"snowflake-sticky-nav-host"},"experiencefragment-sub-header":{"id":"experiencefragment-6cd0723a5f","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/sub-navigation/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/sub-navigation/master.xfmodel.json"},"responsivegrid":{"columnClassNames":{"container_breadcrumb":"aem-GridColumn aem-GridColumn--default--12","container_main_content":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnCount":12,":items":{"container_breadcrumb":{"columnClassNames":{"breadcrumb":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"blog-page-breadcrumb-indentation",":type":"snowflake-site/components/container","appliedCssClassNames":"snowflake-container",":items":{"breadcrumb":{"id":"breadcrumb-be6d8d816e","breadcrumbItems":[{"title":"Blog","path":"/en/blog/","active":false},{"title":"Data Engineering","active":false},{"title":"Stream to Apache Iceberg Easily with Snowpipe Streaming","path":"/en/blog/snowpipe-streaming-iceberg-tables/","active":false}],":type":"snowflake-site/components/blog/breadcrumb"}},":itemsOrder":["breadcrumb"]},"container_main_content":{"columnClassNames":{"flexible_column_container":"aem-GridColumn aem-GridColumn--default--12","related_content":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"main-content",":type":"snowflake-site/components/container","appliedCssClassNames":"snowflake-container",":items":{"flexible_column_container":{"id":"flexible-column-container-897cfdd778","propertiesId":"snowflake-blog-template-main-container","type":"2-column-60-40","alignColumns":"top","containerMaxWidth":"extra-large","topPadding":"none","bottomPadding":"none","spaceBetween":"none","reverseOnMobile":true,"carouselOnMobile":false,"backgroundImageOption":"none","flexible_column_content_container_1":{"layout":"SIMPLE","id":"container-f1376395c0",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"container_hero":{"columnClassNames":{"blog_hero":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"container-bdfb3d6d5b",":type":"snowflake-site/components/container",":items":{"blog_hero":{"id":"blog-hero-57d533dbdf","showClaude":true,"showChatGpt":true,"timeToRead":"7","publicationDate":"AUG 24, 2026","image":{"id":"image-eb8d7f922d","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--5802ecd3-9104-41ce-ba1d-b68593bb9c70/sf-eng-blog-ml-2.png?preferwebp=true&quality=85","lazyEnabled":true,"height":"720","width":"1680",":type":"snowflake-site/components/image"},"authors":[{"authorImage":{"id":"image-dc37522824","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--ebf3baf3-b763-4b40-9263-11d6bc388488/dave-matthews.jpg?preferwebp=true&quality=85","alt":"Headshot of Dave Matthews","lazyEnabled":true,"height":"192","width":"192",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-e42bed6847","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/dave-matthews/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Dave Matthews"}}],"tag":{"tagText":"Data Engineering","tagColor":"#29B5E8"},"title":{"lines":["Stream to Apache Iceberg Easily with Snowpipe Streaming"],"type":"heading2",":type":"snowflake-site/components/title-v2"},":type":"snowflake-site/components/blog/blog-hero"}},":itemsOrder":["blog_hero"]},"responsivegrid_content":{"columnClassNames":{"blog_component_29":"aem-GridColumn aem-GridColumn--default--12","blog_component_26":"aem-GridColumn aem-GridColumn--default--12","blog_component_25":"aem-GridColumn aem-GridColumn--default--12","blog_component_24":"aem-GridColumn aem-GridColumn--default--12","blog_component_23":"aem-GridColumn aem-GridColumn--default--12","blog_component_11":"aem-GridColumn aem-GridColumn--default--12","blog_component_10":"aem-GridColumn aem-GridColumn--default--12","blog_component_32":"aem-GridColumn aem-GridColumn--default--12","blog_component_31":"aem-GridColumn aem-GridColumn--default--12","blog_component_30":"aem-GridColumn aem-GridColumn--default--12","blog_text":"aem-GridColumn aem-GridColumn--default--12","blog_component_1":"aem-GridColumn aem-GridColumn--default--12","blog_component_2":"aem-GridColumn aem-GridColumn--default--12","blog_component_0":"aem-GridColumn aem-GridColumn--default--12","blog_component_5":"aem-GridColumn aem-GridColumn--default--12","blog_component_6":"aem-GridColumn aem-GridColumn--default--12","blog_component_3":"aem-GridColumn aem-GridColumn--default--12","blog_component_4":"aem-GridColumn aem-GridColumn--default--12","blog_component_9":"aem-GridColumn aem-GridColumn--default--12","blog_component_7":"aem-GridColumn aem-GridColumn--default--12","blog_component_17":"aem-GridColumn aem-GridColumn--default--12","blog_component_8":"aem-GridColumn aem-GridColumn--default--12","blog_component_16":"aem-GridColumn aem-GridColumn--default--12","blog_component_14":"aem-GridColumn aem-GridColumn--default--12","blog_component_13":"aem-GridColumn aem-GridColumn--default--12","blog_component_12":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnCount":12,"appliedCssClassNames":"snowflake-layout-container-inner-padding-small",":items":{"blog_component_0":{"id":"blog-title-e42bfea566","propertiesId":"full-scale-streaming-evaluation-without-the-infrastructure-lift","type":"heading2","lines":["Full-scale streaming evaluation without the infrastructure lift"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_1":{"id":"blog-text-eda772b45b","text":"\u003Cp\u003E\u003Cstrong\u003EFor a team trying to find out whether Snowpipe Streaming holds up at 1M+ TPS, you want the shortest path to a real answer.\u003C/strong\u003E Streaming at scale has a reputation. It's the workload teams put off, because standing it up entails S3 buckets, an EKS cluster, Kafka topics, IAM policies, a security review and sign-off from three platform teams who each have their own backlogs. That setup can require additional infrastructure provisioning and coordination before testing begins. Proving these pipelines work at the scale and handle production volume is usually its own project.\u003C/p\u003E\u003Cp\u003EIn our demo environment, we were able to set up and run this evaluation in an afternoon. You can use our \u003Ca href=\"https://www.snowflake.com/en/product/features/snowpipe-streaming/\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESnowpipe Streaming High-Performance Architecture\u003C/a\u003E to stream data into \u003Ca href=\"https://www.snowflake.com/en/developers/guides/apache-iceberg-snowflake-open-catalog-snowpipe-streaming/\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EApache Iceberg™ format\u003C/a\u003E at over 1M TPS, entirely inside Snowflake infrastructure. A container running in Snowpark Container Services (SPCS) generates the load, streams it through the SDK and lands it in a Snowflake-managed Iceberg table. That makes it ready to query in Snowflake right away, with role-based access control (RBAC), lineage and masking all applied as it arrives. In our test environment, we stood up the demo, tested it end to end, measured throughput and tore it down in an afternoon, as we did in our test.\u003C/p\u003E\u003Cp\u003EThis post will walk you through how to actually build an end to end streaming to Iceberg demo, using this \u003Ca href=\"https://github.com/sfc-gh-damatthews/streaming-snowpipe-to-iceberg\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EGitHub Repo\u003C/a\u003E. This example is in python but you can test the SDK options (Java, Python, Node, REST), so that you can credibly recommend this to your business with real results to show.\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_2":{"id":"blog-title-1d003368bf","propertiesId":"architecture-diagram-the-pattern","type":"heading2","lines":["Architecture diagram: The pattern"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_3":{"id":"blog-text-f59a98ef97","text":"\u003Cp\u003EWe wanted to build a demo where everything runs inside Snowflake infrastructure. Multiple Docker containers running in Snowpark Container Services (\u003Ca href=\"https://docs.snowflake.com/en/developer-guide/snowpark-container-services/overview\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESPCS\u003C/a\u003E) generate synthetic test data at your target transactions per second and stream it using the \u003Ca href=\"https://docs.snowflake.com/en/user-guide/snowpipe-streaming/data-load-snowpipe-streaming-overview\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESnowpipe Streaming SDK\u003C/a\u003E. It lands in a \u003Ca href=\"https://docs.snowflake.com/en/user-guide/tables-iceberg-internal-storage\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESnowflake-managed Iceberg table\u003C/a\u003E: open format, queryable within seconds of landing, governed in Snowflake. Authentication is easy: SPCS injects a short-lived OAuth token into the container telling the Streaming SDK to use it. You don't need to manage any secrets.\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_4":{"id":"image-ebcb7144ac","isLcpImage":true,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--94014a2a-306a-49d0-a133-21732f12de61/snowflake-architecture-diagram-showing-spcs-service-load.png?preferwebp=true&quality=85","alt":"Snowflake architecture diagram showing SPCS Service load generator streaming data to Iceberg Table via Snowpipe","lazyEnabled":true,"height":"676","width":"1508","title":"Figure 1: Snowpark Container Service to Iceberg",":type":"snowflake-site/components/image"},"blog_component_5":{"id":"blog-title-a1c5c0f1c3","propertiesId":"why-spcs-to-generate-the-load","type":"heading2","lines":["Why SPCS to generate the load?"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_6":{"id":"blog-text-779d800d1c","text":"\u003Cp\u003ETo be clear, Snowpipe Streaming doesn't need SPCS. The SDK runs anywhere: your laptop, an EC2 box, a Kubernetes pod.\u003C/p\u003E\u003Cp\u003EPutting it in SPCS is what gets you to the test faster since the service runs entirely on infrastructure Snowflake already manages, with nothing new to provision or additional sign-offs to manage for the test itself.\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_7":{"id":"blog-title-296382d7ca","propertiesId":"landing-directly-into-iceberg","type":"heading2","lines":["Landing directly into Iceberg"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_8":{"id":"blog-text-30f3e30586","text":"\u003Cp\u003EThe target is a Snowflake-managed Iceberg table. Rows stream in through the SDK and land as Parquet with Iceberg metadata, managed by Snowflake. Point it at your own S3 external volume later if you want, or leave it managed after the evaluation. The code stays the same either way.\u003C/p\u003E\u003Cp\u003EFor the evaluation it means less setup. A table on Snowflake managed storage skips the external volume, storage integration and cloud storage permissions you'd otherwise have to configure first.\u003C/p\u003E\u003Cp\u003EWhen you're ready to productionize, you've got three clear paths:\u003C/p\u003E\u003Cul\u003E\u003Cli\u003E\u003Cstrong\u003ESnowflake-managed Iceberg with an external volume:\u003C/strong\u003E Parquet files are stored in your S3 bucket. An external engine reads the Parquet files directly from S3, using the Horizon REST Catalog for metadata.\u003C/li\u003E\u003Cli\u003E\u003Cstrong\u003ESnowflake-managed Iceberg on Snowflake storage:\u003C/strong\u003E The files are in Snowflake's internal storage. This option avoids configuring an external volume and associated cloud storage. Leverages Iceberg features for compatibility going forward.\u003C/li\u003E\u003Cli\u003E\u003Cstrong\u003ENative Snowflake table:\u003C/strong\u003E No separately managed external storage infrastructure is required for this configuration.\u003C/li\u003E\u003C/ul\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_9":{"id":"blog-title-c866669752","propertiesId":"demo-see-it-running","type":"heading2","lines":["Demo: See it running"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_10":{"id":"image-ca22069e3d","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--926445a1-e860-4303-bf37-891187d6e11d/terminal-output-showing-snowpipe-streaming-data-ingestion.png?preferwebp=true&quality=85","alt":"Terminal output showing Snowpipe Streaming data ingestion progress from configuration through verification stages","lazyEnabled":true,"height":"1158","width":"1023","title":"Figure 2: Demo of end to end streaming",":type":"snowflake-site/components/image"},"blog_component_11":{"id":"blog-text-0b0929149d","text":"\u003Cp\u003EWhen you run the demo script you can see the SPCS service starts, the producer begins generating the load, and the consumer streams it into the Iceberg table.\u003C/p\u003E\u003Cp\u003EYou can switch to Snowsight, and see the row count climb here as well — millions of rows, queryable as they land.\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_12":{"id":"blog-title-1a6220bbf6","propertiesId":"the-code-building-it-yourself","type":"heading2","lines":["The code: Building it yourself"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_13":{"id":"blog-text-c127b47f1a","text":"\u003Cp\u003ETo help understand how this all works, here is some pseudocode to help visualize how this works. The full code is in the repo, and there are several alternative demos linked below.\u003C/p\u003E\u003Cp\u003EStart with a Snowflake-managed Iceberg table and a pipe:\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_14":{"id":"code-snippet-81118aeafd","language":"sql","codeSnippet":"-- 1. The target table\r\nCREATE OR REPLACE ICEBERG TABLE events (\r\n    event_id STRING,\r\n    event_ts TIMESTAMP_NTZ,\r\n    payload  VARIANT\r\n)\r\nCATALOG = 'SNOWFLAKE'\r\nBASE_LOCATION = 'events/'\r\nICEBERG_VERSION = 3;\r\n\r\n\r\n-- 2. The streaming pipe (extracts typed fields from the SDK's VARIANT payload)\r\nCREATE OR REPLACE PIPE events_pipe\r\nAS COPY INTO events (event_id, event_ts, payload)\r\nFROM (\r\n    SELECT $1:event_id::STRING,\r\n           $1:event_ts::TIMESTAMP_NTZ,\r\n           $1:payload::VARIANT\r\n    FROM TABLE(DATA_SOURCE(TYPE =\u003E 'STREAMING'))\r\n);\r\n","multiLine":true,":type":"snowflake-site/components/code-snippet"},"blog_component_16":{"id":"blog-text-8826340579","text":"\u003Cp\u003ESnowpipe Streaming’s high-performance architecture supports both v2 and v3, but omitting the parameter defaults the table to v2.\u003C/p\u003E\r\n\u003Cp\u003ENext, the consumer:\u003C/p\u003E\r\n","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_17":{"id":"code-snippet-02a9f8cb64","language":"python","codeSnippet":"from snowflake.ingest.streaming import StreamingIngestClient\r\n\r\n\r\n# SPCS injects credentials automatically - no keys, no secrets\r\nprops = {\r\n    \"account\": \"YOUR_ACCOUNT\",\r\n    \"user\": \"YOUR_USER\",\r\n    \"role\": \"STREAMING_SERVICE_ROLE\",\r\n    \"url\": \"https://YOUR_ACCOUNT.snowflakecomputing.com\",\r\n    \"authorization_type\": \"SPCS\",\r\n    \"spcs_token_path\": \"/snowflake/session/token\",\r\n}\r\n\r\n\r\n# Connect to the pipe\r\nclient = StreamingIngestClient(\r\n    client_name=\"my_consumer\",\r\n    db_name=\"STREAMING_DEMO\",\r\n    schema_name=\"PUBLIC\",\r\n    pipe_name=\"events_pipe\",\r\n    properties=props,\r\n)\r\n\r\n\r\n# Open a channel and stream rows\r\nchannel, status = client.open_channel(\"ch_1\")\r\n\r\n\r\nfor i, row in enumerate(generate_load()):\r\n    channel.append_row(row, offset_token=str(i))\r\n\r\n\r\n# Close cleanly\r\nchannel.close()\r\nclient.close()\r\n","multiLine":true,":type":"snowflake-site/components/code-snippet"},"blog_component_23":{"id":"blog-text-636e9e85cb","text":"\u003Cp\u003EThen you deploy the consumer to SPCS with a short service spec. And you watch it land:\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_24":{"id":"code-snippet-0a2e4a1307","language":"sql","codeSnippet":"SELECT\r\nCOUNT(*) AS rows_landed,\r\nMAX(event_ts) AS latest_event\r\nFROM\r\nstreaming_demo.public.events;","multiLine":true,":type":"snowflake-site/components/code-snippet"},"blog_component_25":{"id":"blog-text-1b067f56c3","text":"\u003Cp\u003EJava, Node and the REST interface follow the same shape: open a channel, insert rows, flush. There's a working example for each in the repo, so you can test whichever matches your stack.\u003C/p\u003E\u003Cp\u003ETo try this yourself you can check out the repo.\u003C/p\u003E\u003Cp\u003EBut be aware that the Streaming SDK uses a separate ingest endpoint over HTTPS, so even if you are running inside Snowflake you need to grant an external access integration that allows egress back to \u003Ccode\u003E*.snowflakecomputing.com\u003C/code\u003E.\u003C/p\u003E","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_26":{"id":"blog-title-6a8afaba27","propertiesId":"best-practices","type":"heading2","lines":["Best practices"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_29":{"id":"blog-title-cb23b6d0a7","propertiesId":"snowpipe-streaming-batches-for-you","type":"heading3","lines":["Snowpipe Streaming batches for you"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_30":{"id":"blog-text-737924b36e","text":"\u003Cp\u003ESnowpipe Streaming doesn't write a file per row. The SDK sends rows as soon as you call \u003Ccode\u003EappendRows\u003C/code\u003E, but a server-side buffering tier absorbs them and decides commit timing on its own, batching under the hood so you're never shipping a file per row.\u003C/p\u003E\r\n\u003Cp\u003EOn top of that, Snowflake runs automatic compaction in the background for \u003Ca href=\"https://docs.snowflake.com/en/user-guide/tables-iceberg-internal-storage\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESnowflake-managed Iceberg tables\u003C/a\u003E where Snowflake is the sole writer: Small Parquet files get merged into larger ones, and small manifests get compacted too. You don't schedule it or run it; it's bundled into normal operation.\u003C/p\u003E\r\n\u003Cp\u003EWhat you can do:\u003C/p\u003E\r\n\u003Cul\u003E\r\n\u003Cli\u003E\u003Cb\u003EBatch rows client-side before calling\u003C/b\u003E \u003Ccode\u003EappendRows\u003C/code\u003E: Sending rows one at a time still means one roundtrip per row; batching them into a single call amortizes that overhead, the same principle behind Snowflake's own guidance to compress and send more data per request.\u003C/li\u003E\r\n\u003Cli\u003E\u003Cb\u003EKeep channels long-lived:\u003C/b\u003E Open a channel once per source partition and leave it open for the life of the job instead of opening and closing per micro-batch. It's Snowflake's documented \u003Ca href=\"https://docs.snowflake.com/en/user-guide/snowpipe-streaming/snowpipe-streaming-high-performance-best-practices\" target=\"_blank\" rel=\"noopener noreferrer\"\u003Ebest practice\u003C/a\u003E \u003Ca href=\"https://docs.snowflake.com/en/user-guide/snowpipe-streaming/snowpipe-streaming-high-performance-best-practices\" target=\"_blank\" rel=\"noopener noreferrer\"\u003Efor Snowpipe Streaming\u003C/a\u003E, and it cuts overhead without fighting the batching already happening underneath.\u003C/li\u003E\r\n\u003Cli\u003E\u003Cb\u003ESet\u003C/b\u003E \u003Ccode\u003ETARGET_FILE_SIZE\u003C/code\u003E \u003Cb\u003Eon the table:\u003C/b\u003E This is a table-level property, independent of the SDK, that tells Snowflake what size to target for both new writes and background compaction, no matter how the data got there.\u003C/li\u003E\r\n\u003Cli\u003E\u003Cb\u003ELet throughput do the work:\u003C/b\u003E The higher your sustained TPS, the faster the server-side buffer fills and flushes on its own, which effectively results in a trade off between throughput and latency for a given file size. \u003C/li\u003E\r\n\u003C/ul\u003E\r\n","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_component_31":{"id":"blog-title-2da2601125","propertiesId":"whats-coming-iceberg-v4","type":"heading2","lines":["What's coming: Iceberg v4"],":type":"snowflake-site/components/blog/blog-title"},"blog_component_32":{"id":"blog-text-bce9d4e6e5","text":"\u003Cp\u003ESnowpipe Streaming and automatic compaction address the physical small-files problem, but frequent commits can still create overhead in Iceberg’s metadata tree. Each commit currently writes a new metadata JSON, manifest list and manifest — even when it contains only one small file. For workloads that commit every few seconds, that repeated metadata work can become the bottleneck.\u003C/p\u003E\r\n\u003Cp\u003E\u003Cspan style=\"font-family: adobe-clean, "Source Sans Pro", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Ubuntu, "Trebuchet MS", "Lucida Grande", sans-serif;\"\u003EThe Iceberg v4 proposal introduces an Adaptive Metadata Tree to reduce that write amplification. Instead of creating a new manifest for every small commit, the root manifest can inline those commits. A single Parquet write and an atomic pointer swap replace the current chain, keeping metadata I/O per commit constant rather than scaling with manifest count. For high-frequency streaming workloads, that can dramatically improve streaming latency.\u003C/span\u003E\u003C/p\u003E\r\n\u003Cp\u003E\u003Cspan style=\"font-family: adobe-clean, "Source Sans Pro", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Ubuntu, "Trebuchet MS", "Lucida Grande", sans-serif;\"\u003EStreaming is one of the hero scenarios for Iceberg v4: Workloads that commit every few seconds expose metadata-write amplification directly, making the benefits of the Adaptive Metadata Tree especially relevant. The v4 spec is still evolving, but it’s worth watching if Iceberg streaming is on your roadmap. Snowflake is proud to participate actively in shaping the future of the format.\u003C/span\u003E\u003C/p\u003E\r\n\u003Ch3\u003EWhat this means for your architecture\u003C/h3\u003E\r\n","richText":true,":type":"snowflake-site/components/blog/blog-text"},"blog_text":{"id":"blog-text-4292f74e82","text":"\u003Cp\u003EIn this post, we demonstrated how effective Snowpipe Streaming into Iceberg can be. At the kind of load at which a serious media or telco platform runs, with no S3 bucket, EKS cluster, Kafka topic, or IAM role required for this test setup, the workload ran inside Snowflake and could be torn down after the evaluation.\u003C/p\u003E\r\n\u003Cp\u003EThe infrastructure that normally sits between a team and a streaming test, the buckets and clusters and auth and sign-offs, isn't a prerequisite. You can answer, "Does this meet our requirements quickly?".\u003C/p\u003E\r\n\u003Cp\u003EIf you're on the Site Reliability Engineering (SRE) or Data Engineering side of a large media or telco platform and Iceberg streaming is on the roadmap, start with the test, not the procurement cycle. We invite you to clone the repo, deploy to SPCS and watch it run.\u003C/p\u003E\r\n\u003Cul\u003E\r\n\u003Cli\u003ETry it now with the \u003Ca href=\"https://github.com/sfc-gh-damatthews/streaming-snowpipe-to-iceberg\"\u003EGitHub repo\u003C/a\u003E\u003C/li\u003E\r\n\u003Cli\u003EGo deeper by exploring the \u003Ca href=\"https://docs.snowflake.com/en/user-guide/snowpipe-streaming/snowpipe-streaming-high-performance-iceberg\"\u003ESnowpipe Streaming High-Performance Architecture documentation\u003C/a\u003E\u003C/li\u003E\r\n\u003C/ul\u003E\r\n","richText":true,":type":"snowflake-site/components/blog/blog-text"}},":itemsOrder":["blog_component_0","blog_component_1","blog_component_2","blog_component_3","blog_component_4","blog_component_5","blog_component_6","blog_component_7","blog_component_8","blog_component_9","blog_component_10","blog_component_11","blog_component_12","blog_component_13","blog_component_14","blog_component_16","blog_component_17","blog_component_23","blog_component_24","blog_component_25","blog_component_26","blog_component_29","blog_component_30","blog_component_31","blog_component_32","blog_text"],":type":"wcm/foundation/components/responsivegrid"},"responsivegrid_premium_content_banner":{"columnClassNames":{},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","columnCount":12,"appliedCssClassNames":"snowflake-responsive-component-top-padding-medium",":items":{},":itemsOrder":[],":type":"wcm/foundation/components/responsivegrid"},"container_author_chip":{"columnClassNames":{"author_chip":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"container-8331de78c8",":type":"snowflake-site/components/container","appliedCssClassNames":"snowflake-responsive-component-top-padding-medium",":items":{"author_chip":{"id":"author-chip-c1dc9b6039","title":{"id":"title","type":"heading2","lines":["Learn more about the authors"],":type":"snowflake-site/components/title-v2"},"authors":[{"authorImage":{"id":"image-dc37522824","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--ebf3baf3-b763-4b40-9263-11d6bc388488/dave-matthews.jpg?preferwebp=true&quality=85","alt":"Headshot of Dave Matthews","lazyEnabled":true,"height":"192","width":"192",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-e42bed6847","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/dave-matthews/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Dave Matthews"},"authorTitle":"Senior Solution Engineer"}],":type":"snowflake-site/components/blog/author-chip"}},":itemsOrder":["author_chip"]},"container_share_article":{"columnClassNames":{"share_article":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"container-f6ae9a8e0f",":type":"snowflake-site/components/container","appliedCssClassNames":"snowflake-responsive-component-top-padding-small",":items":{"share_article":{"id":"share-article-642b8a4552","linkedInShareUrl":"https://www.linkedin.com/shareArticle?mini=true&url=https%3A%2F%2Fwww.snowflake.com%2Fcontent%2Fsnowflake-site%2Fglobal%2Fen%2Fblog%2Fsnowpipe-streaming-iceberg-tables&title=Stream+to+Apache+Iceberg+Easily+with+Snowpipe+Streaming","twitterShareUrl":"https://x.com/intent/post?url=https%3A%2F%2Fwww.snowflake.com%2Fcontent%2Fsnowflake-site%2Fglobal%2Fen%2Fblog%2Fsnowpipe-streaming-iceberg-tables&text=Stream+to+Apache+Iceberg+Easily+with+Snowpipe+Streaming","facebookShareUrl":"https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fwww.snowflake.com%2Fcontent%2Fsnowflake-site%2Fglobal%2Fen%2Fblog%2Fsnowpipe-streaming-iceberg-tables",":type":"snowflake-site/components/blog/share-article"}},":itemsOrder":["share_article"]}},":itemsOrder":["container_hero","responsivegrid_content","responsivegrid_premium_content_banner","container_author_chip","container_share_article"]},"flexible_column_content_container_2":{"layout":"SIMPLE","id":"container-95b9304f51",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"blog_table_of_content":{"id":"blog-table-of-content-8f3f6f93e1","tableOfContents":[],":type":"snowflake-site/components/blog/blog-table-of-content"}},":itemsOrder":["blog_table_of_content"]},":type":"snowflake-site/components/flexible-column-container","isBlogPage":true,"isActiveTOC":false},"related_content":{"id":"related-content-b65c51bb9d","relatedContent":[{"configurationStatus":{"configured":true,"message":""},"tags":[{"tagText":"Product and Technology","tagColor":"#3F808A"}],"publicationDate":"JUL 30, 2026","authors":[{"authorImage":{"id":"image-927431d9ad","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--c83c4010-5613-4c44-a706-5628c1e8717a/hannah-miao.png?preferwebp=true&quality=85","alt":"Headshot-style photo of Hannah Miao of Snowflake","lazyEnabled":true,"height":"800","width":"800",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-5748890fd9","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/hannah-miao/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Hannah Miao"}},{"authorImage":{"id":"image-690a8390c1","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--93f604fa-492f-4244-b5fd-e2632ae4cf81/david-zhang.jpg?preferwebp=true&quality=85","alt":"Headshot-style photo of David Zhang of Snowflake","lazyEnabled":true,"height":"3000","width":"2998",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-01e9c0e505","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/david-zhang/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"David Zhang"}}],"title":{"lines":["Unlocking Open Observability with Observe on Apache Iceberg"],"type":"heading4",":type":"snowflake-site/components/title-v2"},"button":{"id":"button-178cfbc7e9","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/observe-apache-iceberg-open-observability/"},"linkTargetContentType":"GENERIC",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Unlocking Open Observability with Observe on Apache Iceberg"},":type":"snowflake-site/components/blog/card"},{"configurationStatus":{"configured":true,"message":""},"tags":[{"tagText":"Data Engineering","tagColor":"#29B5E8"}],"publicationDate":"JAN 30, 2026","authors":[{"authorImage":{"id":"image-669b43f8c2","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--bb3062e2-6de8-4d7c-9094-810ded903737/chasethomas.jpg?preferwebp=true&quality=85","lazyEnabled":true,"height":"400","width":"400",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-cb7e7ff651","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/chase-thomas/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Chase Thomas"}},{"authorImage":{"id":"image-a264f20802","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--a8bebf48-2c53-48f8-94fd-41305c1adf18/praveen-gattu.png?preferwebp=true&quality=85","lazyEnabled":true,"height":"146","width":"142",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-8bfc7aa6ba","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/praveen-gattu/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Praveen Gattu"}}],"title":{"lines":["Migrating to Snowpipe Streaming High Performance Architecture"],"type":"heading4",":type":"snowflake-site/components/title-v2"},"button":{"id":"button-178cfbc7e9","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/migrating-snowpipe-streaming-architecture/"},"linkTargetContentType":"GENERIC",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Migrating to Snowpipe Streaming High Performance Architecture"},":type":"snowflake-site/components/blog/card"},{"configurationStatus":{"configured":true,"message":""},"tags":[{"tagText":"Data Engineering","tagColor":"#29B5E8"}],"publicationDate":"OCT 09, 2025","authors":[{"authorImage":{"id":"image-669b43f8c2","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--bb3062e2-6de8-4d7c-9094-810ded903737/chasethomas.jpg?preferwebp=true&quality=85","lazyEnabled":true,"height":"400","width":"400",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-cb7e7ff651","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/chase-thomas/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Chase Thomas"}},{"authorImage":{"id":"image-22f7d39c65","isLcpImage":false,"src":"https://www.snowflake.com/adobe/dynamicmedia/deliver/dm-aid--49b125aa-1da5-43b3-9515-9ee3e56bf2fd/revi.jpg?preferwebp=true&quality=85","alt":"Revi Cheng","lazyEnabled":true,"height":"512","width":"512",":type":"snowflake-site/components/image"},"authorCta":{"id":"button-a2ad96d8d2","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/authors/revi-cheng/"},"linkTargetContentType":"DOCUMENT_LEARN",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Revi Cheng"}}],"title":{"lines":["Scaling Streaming at Snowflake: Introducing Our Next-Gen Snowpipe Streaming Architecture"],"type":"heading4",":type":"snowflake-site/components/title-v2"},"button":{"id":"button-178cfbc7e9","showOutboundIcon":false,"buttonLink":{"valid":true,"url":"/en/blog/engineering/next-gen-snowpipe-streaming-architecture/"},"linkTargetContentType":"GENERIC",":type":"snowflake-site/components/button","linkType":"SNOWFLAKE_INTERNAL","text":"Scaling Streaming at Snowflake: Introducing Our Next-Gen Snowpipe Streaming Architecture"},":type":"snowflake-site/components/blog/card"}],":type":"snowflake-site/components/blog/related-content","isBlogPage":true}},":itemsOrder":["flexible_column_container","related_content"]}},":itemsOrder":["container_breadcrumb","container_main_content"],":type":"wcm/foundation/components/responsivegrid"},"container_47873732":{"additionalClasses":"section--blog-newsletter","columnClassNames":{"flexible_column_cont":"aem-GridColumn aem-GridColumn--default--12"},"gridClassNames":"aem-Grid aem-Grid--12 aem-Grid--default--12","layout":"RESPONSIVE_GRID","columnCount":12,"id":"container-cc222b5511",":type":"snowflake-site/components/container","appliedCssClassNames":"snowflake-container",":items":{"flexible_column_cont":{"id":"flexible-column-container-e37740df17","type":"1-column","alignColumns":"top","containerMaxWidth":"extra-large","topPadding":"small","bottomPadding":"none","spaceBetween":"small","reverseOnMobile":false,"carouselOnMobile":false,"propertiesCSSClasses":"section--blog-newsletter","backgroundImageOption":"none","flexible_column_content_container_1":{"layout":"SIMPLE","id":"container-b61de5a618",":type":"snowflake-site/components/flexible-column-container/flexible-column-content-container",":items":{"marketo_v2":{"id":"marketo-v2-791779acad","marketoForm":{"successUrl":null,"formId":"3320","edit":false,"hidden":null,"script":null,"values":null},"title":{"id":"title","type":"heading3","lines":["Subscribe to our blog newsletter","Get the best, coolest and latest delivered to your inbox each week"],":type":"snowflake-site/components/title-v2"},"munchkinId":"252-RFO-227","serverInstance":"252-RFO-227.mktoweb.com","partnerCampaignPage":false,"marketoConfigured":true,"formConfigured":true,":type":"snowflake-site/components/form/marketo-v2"},"text":{"id":"text-1821e247d1","additionalClasses":"newsletter-disclaimer","text":"\u003Cp\u003EBy submitting this form, I understand Snowflake will process my personal information in accordance with their Privacy Notice.\u003C/p\u003E\r\n","richText":true,":type":"snowflake-site/components/text"}},":itemsOrder":["marketo_v2","text"]},":type":"snowflake-site/components/flexible-column-container","isBlogPage":true,"isActiveTOC":false}},":itemsOrder":["flexible_column_cont"]},"experiencefragment-pre-footer":{"id":"experiencefragment-606a1a9cd5","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/get-started-pre-footer/get-started-pre-footer/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/get-started-pre-footer/get-started-pre-footer.xfmodel.json"},"markup_editor":{"id":"markup-editor-4023425c12","title":"Page CSS","cssContent":"@media screen and (min-width:768px){.snowflake-blog-author-chip-wrapper{justify-content:flex-start}.snowflake-blog-related-content-on-blog-page{max-width:1408px;margin-left:auto;margin-right:auto}.snowflake-text{font-family:Lato,sans-serif;font-weight:400;font-size:16px;line-height:24px}}.section--blog-newsletter{max-width:none;width:100%;padding-left:0;padding-right:0;margin-left:0;margin-right:0;margin-bottom:0}.section--blog-newsletter .mktoField{background-color:transparent !important}.section--blog-newsletter\u003E.container{padding-left:0;padding-right:0}@media screen and (min-width:768px){.section--blog-newsletter\u003E.container{padding-left:0;padding-right:0}}.newsletter-disclaimer p{font-size:14px !important}.section--blog-newsletter .snowflake-marketo-form-container{margin-bottom:24px;background-color:#f6f9fa;gap:48px;box-shadow:none}.section--blog-newsletter .snowflake-title p.snowflake-title-line:first-child{font-family:Texta;font-size:24px;line-height:26px;font-weight:700;margin-bottom:4px}.section--blog-newsletter .snowflake-title p.snowflake-title-line{text-transform:none;font-family:\"Lato\",sans-serif;font-size:16px;line-height:24px;font-weight:normal}@media screen and (min-width:1024px){.section--blog-newsletter .snowflake-marketo-form-container{display:flex;justify-content:center}.section--blog-newsletter .snowflake-title .snowflake-title-line{text-align:left}.section--blog-newsletter .snowflake-marketo-form .mktoFormRow:has(\u003E input[type=\"hidden\"]){flex-grow:0}.section--blog-newsletter .snowflake-marketo-form{display:flex;width:50% !important}.section--blog-newsletter .snowflake-marketo-form .mktoButtonRow{flex-grow:0;width:auto !important;margin-left:0;margin-right:0}.section--blog-newsletter .snowflake-marketo-form .mktoFormRow{flex-grow:1}.section--blog-newsletter\u003E.container{padding-left:0;padding-right:0}.section--blog-newsletter .snowflake-marketo-form-title{width:50%;margin-bottom:0 !important}.section--blog-newsletter .center .snowflake-title{align-items:flex-start}}.snowflake-sub-navigation a.snowflake-sub-navigation-primary-link{width:auto !important}.snowflake-blog-hero{align-items:stretch !important}","isGSAPEnabled":false,":type":"snowflake-site/components/markup-editor"},"markup_editor-table":{"id":"markup-editor-fbd3a56ae0","title":"Table Styling CSS","cssContent":"#snowflake-blog-template-main-container table{width:100%;background-color:var(--ui-background-01);border-collapse:collapse;border:2px solid var(--ui-background-09);font-family:'Lato',sans-serif;color:var(--ui-background-09)}#snowflake-blog-template-main-container table thead{background-color:var(--ui-01)}#snowflake-blog-template-main-container table th,#snowflake-blog-template-main-container table td{border:2px solid var(--ui-background-09);padding:var(--spacing-01)}","isGSAPEnabled":false,":type":"snowflake-site/components/markup-editor"},"experiencefragment-footer":{"id":"experiencefragment-c6cfd90842","localizedFragmentVariationPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/footer/master/jcr:content","configured":true,":type":"snowflake-site/components/experiencefragment","xfModelPath":"/content/experience-fragments/snowflake-site/language-masters/en/site/footer/master.xfmodel.json"}},":itemsOrder":["experiencefragment-banner","experiencefragment-header","experiencefragment-sub-header","responsivegrid","container_47873732","experiencefragment-pre-footer","markup_editor","markup_editor-table","experiencefragment-footer"],":type":"wcm/foundation/components/responsivegrid"}},":itemsOrder":["root"],"coveoConfig":{"searchHub":"snowflake.com","organizationId":"snowflakecomputingproduction8neljofn","apiKey":"xx335921a6-2a0a-40f2-a167-e390b4766c3d","pipeline":"snowflake.com"}}