diff --git a/acceptance/multihomed/BUILD.bazel b/acceptance/multihomed/BUILD.bazel new file mode 100644 index 0000000000..78ad13cc02 --- /dev/null +++ b/acceptance/multihomed/BUILD.bazel @@ -0,0 +1,15 @@ +load("//acceptance/common:topogen.bzl", "topogen_test") + +topogen_test( + name = "test", + src = "test.py", + args = [ + "--executable=test-client:$(location //acceptance/multihomed/test-client)", + "--executable=test-server:$(location //acceptance/multihomed/test-server)", + ], + data = [ + "//acceptance/multihomed/test-client", + "//acceptance/multihomed/test-server", + ], + topo = "//topology:tiny.topo", +) diff --git a/acceptance/multihomed/test-client/BUILD.bazel b/acceptance/multihomed/test-client/BUILD.bazel new file mode 100644 index 0000000000..8d83cf7245 --- /dev/null +++ b/acceptance/multihomed/test-client/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/scionproto/scion/acceptance/multihomed/test-client", + visibility = ["//visibility:private"], + deps = [ + "//pkg/daemon:go_default_library", + "//pkg/daemon/types:go_default_library", + "//pkg/snet:go_default_library", + ], +) + +go_binary( + name = "test-client", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/acceptance/multihomed/test-client/main.go b/acceptance/multihomed/test-client/main.go new file mode 100644 index 0000000000..6cf20c5df3 --- /dev/null +++ b/acceptance/multihomed/test-client/main.go @@ -0,0 +1,127 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "log" + "os" + + "github.com/scionproto/scion/pkg/daemon" + daemontypes "github.com/scionproto/scion/pkg/daemon/types" + "github.com/scionproto/scion/pkg/snet" +) + +func main() { + log.SetOutput(os.Stdout) + + // Parse test inputs. The remote is provided as a full SCION UDP address so the same + // client binary can probe server primary and secondary IPs without code changes. + var daemonAddr string + var localAddr snet.UDPAddr + var remoteAddr snet.UDPAddr + var expect string + var expectAddr *snet.UDPAddr + + flag.StringVar(&daemonAddr, "daemon", "", "SCION daemon address") + flag.Var(&localAddr, "local", "Local SCION address") + flag.Var(&remoteAddr, "remote", "Remote SCION address") + flag.StringVar(&expect, "expect", "", "Expected remote SCION address") + flag.Parse() + + if expect != "" { + parsed, err := snet.ParseUDPAddr(expect) + if err != nil { + log.Fatalf("parse expected remote address: %v", err) + } + expectAddr = parsed + } + + if daemonAddr == "" { + daemonAddr = os.Getenv("SCION_DAEMON_ADDRESS") + } + if daemonAddr == "" { + daemonAddr = os.Getenv("SCION_DAEMON") + } + if daemonAddr == "" { + log.Fatal("daemon address missing: pass -daemon or set SCION_DAEMON_ADDRESS/SCION_DAEMON") + } + + // Resolve a path from local IA to remote IA. + ctx := context.Background() + sd, err := daemon.NewService(daemonAddr).Connect(ctx) + if err != nil { + log.Fatalf("connect daemon: %v", err) + } + defer sd.Close() + + paths, err := sd.Paths(ctx, remoteAddr.IA, localAddr.IA, daemontypes.PathReqFlags{Refresh: true}) + if err != nil { + log.Fatalf("path lookup: %v", err) + } + if len(paths) == 0 { + log.Fatalf("no path from %s to %s", localAddr.IA, remoteAddr.IA) + } + sp := paths[0] + + // Build a SCION connection pinned to the selected path. + topo, err := daemon.LoadTopology(ctx, sd) + if err != nil { + log.Fatalf("load topology: %v", err) + } + remoteAddr.Path = sp.Dataplane() + remoteAddr.NextHop = sp.UnderlayNextHop() + + sn := snet.SCIONNetwork{ + Topology: topo, + SCMPHandler: snet.DefaultSCMPHandler{ + RevocationHandler: daemon.RevHandler{Connector: sd}, + }, + } + + conn, err := sn.Dial(ctx, "udp", localAddr.Host, &remoteAddr) + if err != nil { + log.Fatalf("dial: %v", err) + } + defer conn.Close() + + // Exchange ping/pong payloads and assert reply endpoint if requested by the caller. + _, err = conn.Write([]byte("ping")) + if err != nil { + log.Fatalf("write ping: %v", err) + } + + buf := make([]byte, 2048) + n, from, err := conn.ReadFrom(buf) + if err != nil { + log.Fatalf("read pong: %v", err) + } + if string(buf[:n]) != "pong" { + log.Fatalf("unexpected payload: %q", string(buf[:n])) + } + if expectAddr != nil { + got, ok := from.(*snet.UDPAddr) + if !ok { + log.Fatalf("unexpected remote type %T", from) + } + if got.IA != expectAddr.IA || got.Host.Port != expectAddr.Host.Port || + !got.Host.IP.Equal(expectAddr.Host.IP) { + log.Fatalf("unexpected remote. got=%s want=%s", got, expectAddr) + } + } + + log.Printf("client success remote=%s", from) +} diff --git a/acceptance/multihomed/test-server/BUILD.bazel b/acceptance/multihomed/test-server/BUILD.bazel new file mode 100644 index 0000000000..08e5d3a1d9 --- /dev/null +++ b/acceptance/multihomed/test-server/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/scionproto/scion/acceptance/multihomed/test-server", + visibility = ["//visibility:private"], + deps = ["//pkg/snet:go_default_library"], +) + +go_binary( + name = "test-server", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/acceptance/multihomed/test-server/main.go b/acceptance/multihomed/test-server/main.go new file mode 100644 index 0000000000..302563a46c --- /dev/null +++ b/acceptance/multihomed/test-server/main.go @@ -0,0 +1,102 @@ +// Copyright 2026 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "log" + "net" + "os" + "strconv" + + "github.com/scionproto/scion/pkg/snet" +) + +func main() { + log.SetOutput(os.Stdout) + + // Parse test inputs. The same server binary is used for: + // - IPv4 unbound mode: bind to 0.0.0.0 + // - IPv6 unbound mode: bind to :: + var bindAddr string + var port int + + flag.StringVar(&bindAddr, "bind", "0.0.0.0", "Bind host") + flag.IntVar(&port, "port", 31000, "Bind UDP port") + flag.Parse() + + // Bind a raw UDP socket in the tester namespace. Replies are created by reversing the + // received SCION packet, which preserves the destination address the client originally used. + local, err := net.ResolveUDPAddr("udp", net.JoinHostPort(bindAddr, portString(port))) + if err != nil { + log.Fatalf("parse bind address: %v", err) + } + conn, err := net.ListenUDP("udp", local) + if err != nil { + log.Fatalf("listen: %v", err) + } + defer conn.Close() + + log.Printf("server running bind=%s:%d", bindAddr, port) + + // Single ping/pong exchange; process exits afterwards so the test can restart with a fresh bind. + var pkt snet.Packet + pkt.Prepare() + n, lastHop, err := conn.ReadFrom(pkt.Bytes) + if err != nil { + log.Fatalf("read ping: %v", err) + } + pkt.Bytes = pkt.Bytes[:n] + + if err := pkt.Decode(); err != nil { + log.Fatalf("decode packet: %v", err) + } + pld, ok := pkt.Payload.(snet.UDPPayload) + if !ok { + log.Fatalf("unexpected payload type %T", pkt.Payload) + } + if string(pld.Payload) != "ping" { + log.Fatalf("unexpected payload: %q", string(pld.Payload)) + } + + rawPath, ok := pkt.Path.(snet.RawPath) + if !ok { + log.Fatalf("unexpected path type %T", pkt.Path) + } + replyPath, err := snet.DefaultReplyPather{}.ReplyPath(rawPath) + if err != nil { + log.Fatalf("reverse path: %v", err) + } + + pkt.Destination, pkt.Source = pkt.Source, pkt.Destination + pkt.Path = replyPath + pkt.Payload = snet.UDPPayload{ + SrcPort: pld.DstPort, + DstPort: pld.SrcPort, + Payload: []byte("pong"), + } + if err := pkt.Serialize(); err != nil { + log.Fatalf("serialize reply: %v", err) + } + if _, err := conn.WriteTo(pkt.Bytes, lastHop); err != nil { + log.Fatalf("write pong: %v", err) + } + + log.Printf("served ping from %s", pkt.Destination) +} + +func portString(port int) string { + return strconv.Itoa(port) +} diff --git a/acceptance/multihomed/test.py b/acceptance/multihomed/test.py new file mode 100644 index 0000000000..671330f717 --- /dev/null +++ b/acceptance/multihomed/test.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# Copyright 2026 ETH Zurich +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Acceptance test for multihomed endhost source-address selection. + +Manual runs: + bazel test --config=integration //acceptance/multihomed:test --test_output=streamed + +The setup/run/teardown stages can also be exercised independently: + bazel run //acceptance/multihomed:test_setup + bazel run //acceptance/multihomed:test_run + bazel run //acceptance/multihomed:test_teardown + +Test flow: +1. Generate the tiny topology and keep the generated control-plane addressing unchanged. +2. Attach one extra compose-managed subnet to `br1-ff00_0_110-2` and to the AS110 tester + namespace so the server gets a second address behind the second border router without + rewriting the topology generator's internal BR addresses. +3. Start fresh unbound server instances in AS110 and probe them from AS111 and AS112, + targeting the address that is reachable through each border router. +4. Start fresh bound server instances in AS110, one bound to the AS111-facing address + and one bound to the AS112-facing address. +5. Require the client in all four cases to observe the reply coming back from the exact + SCION source address it targeted. +""" + +import time + +import yaml +from plumbum import local + +from acceptance.common import base + + +SERVER_PORT = 31000 +SERVER_PRIMARY_IP = "172.20.0.22" +SERVER_SECONDARY_IP = "192.168.201.3" +SERVER_SECONDARY_BR_IP = "192.168.201.2" +SERVER_SECONDARY_SUBNET = "192.168.201.0/24" +SERVER_SECONDARY_NETWORK = "local_110_br2" + +SERVER_IA = "1-ff00:0:110" +CLIENT_111_IA = "1-ff00:0:111" +CLIENT_112_IA = "1-ff00:0:112" + +SERVER_CONTAINER = "tester_1-ff00_0_110" +CLIENT_111_CONTAINER = "tester_1-ff00_0_111" +CLIENT_112_CONTAINER = "tester_1-ff00_0_112" +SERVER_DISPATCHER = "disp_tester_1-ff00_0_110" +SERVER_BR2 = "br1-ff00_0_110-2" + + +class Test(base.TestTopogen): + def setup_prepare(self): + super().setup_prepare() + + # Add a second endhost-facing subnet behind br1-ff00_0_110-2 while leaving the + # generated control-plane topology untouched. This is intentionally a compose-only + # network change: the BR keeps its original generated internal address for SCION + # control-plane traffic, while the server gains a second reachable host address for + # the multihoming assertion. The raw packet-reversal server keeps the reply source + # equal to the destination address that each client targeted. + compose_path = self.artifacts / "gen/scion-dc.yml" + with open(compose_path, "r", encoding="utf-8") as file: + scion_dc = yaml.safe_load(file) + + scion_dc["networks"][SERVER_SECONDARY_NETWORK] = { + "driver": "bridge", + "ipam": {"config": [{"subnet": SERVER_SECONDARY_SUBNET}]}, + } + scion_dc["services"][SERVER_BR2]["networks"][SERVER_SECONDARY_NETWORK] = { + "ipv4_address": SERVER_SECONDARY_BR_IP, + } + scion_dc["services"][SERVER_DISPATCHER]["networks"][SERVER_SECONDARY_NETWORK] = { + "ipv4_address": SERVER_SECONDARY_IP, + } + + with open(compose_path, "w", encoding="utf-8") as file: + yaml.safe_dump(scion_dc, file, sort_keys=False) + + def _run(self): + self.await_connectivity() + time.sleep(10) + + test_client = local["realpath"](self.get_executable("test-client").executable).strip() + test_server = local["realpath"](self.get_executable("test-server").executable).strip() + self.dc("cp", test_server, f"{SERVER_CONTAINER}:/bin/") + self.dc("cp", test_client, f"{CLIENT_111_CONTAINER}:/bin/") + self.dc("cp", test_client, f"{CLIENT_112_CONTAINER}:/bin/") + + print( + "server IPs configured: primary=%s, secondary=%s" + % (SERVER_PRIMARY_IP, SERVER_SECONDARY_IP) + ) + self._run_scenario( + client_container=CLIENT_111_CONTAINER, + client_ia=CLIENT_111_IA, + remote_ip=SERVER_PRIMARY_IP, + bind_ip="0.0.0.0", + label="AS111 -> AS110 via br1-ff00_0_110-1 with unbound server", + ) + self._run_scenario( + client_container=CLIENT_112_CONTAINER, + client_ia=CLIENT_112_IA, + remote_ip=SERVER_SECONDARY_IP, + bind_ip="0.0.0.0", + # The extra compose-managed subnet sits behind br1-ff00_0_110-2, so the + # reply should be sourced from the second server address on that subnet. + label="AS112 -> AS110 via br1-ff00_0_110-2 with unbound server", + ) + self._run_scenario( + client_container=CLIENT_111_CONTAINER, + client_ia=CLIENT_111_IA, + remote_ip=SERVER_PRIMARY_IP, + bind_ip=SERVER_PRIMARY_IP, + label="AS111 -> AS110 via br1-ff00_0_110-1 with server bound to primary IP", + ) + self._run_scenario( + client_container=CLIENT_112_CONTAINER, + client_ia=CLIENT_112_IA, + remote_ip=SERVER_SECONDARY_IP, + bind_ip=SERVER_SECONDARY_IP, + # The extra compose-managed subnet sits behind br1-ff00_0_110-2, so the + # reply should be sourced from the second server address on that subnet. + label="AS112 -> AS110 via br1-ff00_0_110-2 with server bound to secondary IP", + ) + + def _run_scenario( + self, + client_container: str, + client_ia: str, + remote_ip: str, + bind_ip: str, + label: str, + ): + remote = f"{SERVER_IA},{remote_ip}:{SERVER_PORT}" + print(f"running {label}: {remote}") + self.dc.execute_detached( + SERVER_CONTAINER, + "bash", + "-c", + f'test-server -bind "{bind_ip}" -port {SERVER_PORT}', + ) + time.sleep(3) + result = self.dc.execute( + client_container, + "bash", + "-c", + ( + f'test-client -local "{client_ia},0.0.0.0:0" ' + f'-remote "{remote}" -expect "{remote}"' + ), + ) + print(result) + +if __name__ == "__main__": + base.main(Test) diff --git a/gateway/dataplane/BUILD.bazel b/gateway/dataplane/BUILD.bazel index c65bd346c1..5ab0e9c9d3 100644 --- a/gateway/dataplane/BUILD.bazel +++ b/gateway/dataplane/BUILD.bazel @@ -67,6 +67,7 @@ go_test( "//pkg/private/xtest:go_default_library", "//pkg/snet:go_default_library", "//pkg/snet/mock_snet:go_default_library", + "//pkg/snet/multihomed:go_default_library", "//pkg/snet/path:go_default_library", "//private/ringbuf:go_default_library", "@com_github_golang_mock//gomock:go_default_library", diff --git a/gateway/dataplane/session_test.go b/gateway/dataplane/session_test.go index b315d81fd4..b685b5e941 100644 --- a/gateway/dataplane/session_test.go +++ b/gateway/dataplane/session_test.go @@ -31,6 +31,7 @@ import ( "github.com/scionproto/scion/pkg/private/mocks/net/mock_net" "github.com/scionproto/scion/pkg/snet" "github.com/scionproto/scion/pkg/snet/mock_snet" + "github.com/scionproto/scion/pkg/snet/multihomed" snetpath "github.com/scionproto/scion/pkg/snet/path" ) @@ -83,6 +84,7 @@ func TestTwoPaths(t *testing.T) { } func TestNoLeak(t *testing.T) { + multihomed.StopContinuousCheckInterfaces(t) defer goleak.VerifyNone(t) ctrl := gomock.NewController(t) diff --git a/pkg/snet/BUILD.bazel b/pkg/snet/BUILD.bazel index 4307d2b1b3..a3eaee135b 100644 --- a/pkg/snet/BUILD.bazel +++ b/pkg/snet/BUILD.bazel @@ -38,6 +38,7 @@ go_library( "//pkg/slayers/path/epic:go_default_library", "//pkg/slayers/path/onehop:go_default_library", "//pkg/slayers/path/scion:go_default_library", + "//pkg/snet/multihomed:go_default_library", "//pkg/stun:go_default_library", "//private/topology:go_default_library", "//private/topology/underlay:go_default_library", diff --git a/pkg/snet/conn.go b/pkg/snet/conn.go index ad94cb6199..f72400dea1 100644 --- a/pkg/snet/conn.go +++ b/pkg/snet/conn.go @@ -22,7 +22,6 @@ import ( "github.com/scionproto/scion/pkg/private/common" "github.com/scionproto/scion/pkg/private/ctrl/path_mgmt" - "github.com/scionproto/scion/pkg/private/serrors" "github.com/scionproto/scion/pkg/slayers" ) @@ -58,7 +57,12 @@ type Conn struct { // NewCookedConn returns a "cooked" Conn. The Conn object can be used to // send/receive SCION traffic with the usual methods. // It takes as arguments a non-nil PacketConn and a non-nil Topology parameter. -// Nil or unspecified addresses for the PacketConn object are not supported. +// The local address of the PacketConn can be nil or unspecified, leaving the socket not bound +// to any particular interface; however it has its limitations, namely a created Conn not +// being able to properly react to a routing change in the OS unless the routing change is +// accompanied by a change in the local network interfaces, in the form of changing their +// local IP addresses, or adding/removing one or more local network interfaces. +// // This is an advanced API, that allows fine-tuning of the Conn underlay functionality. // The general methods for obtaining a Conn object are still SCIONNetwork.Listen and // SCIONNetwork.Dial. @@ -72,10 +76,8 @@ func NewCookedConn( IA: topo.LocalIA, Host: pconn.LocalAddr().(*net.UDPAddr), } - if local.Host == nil || local.Host.IP.IsUnspecified() { - return nil, serrors.New("nil or unspecified address is not supported.") - } hasSTUN := hasSTUNConn(pconn) + return &Conn{ conn: pconn, local: local, diff --git a/pkg/snet/multihomed/BUILD.bazel b/pkg/snet/multihomed/BUILD.bazel new file mode 100644 index 0000000000..4cf73c47d4 --- /dev/null +++ b/pkg/snet/multihomed/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library") +load("//tools:go.bzl", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "interfaces.go", + "outbound_addr.go", + ], + importpath = "github.com/scionproto/scion/pkg/snet/multihomed", + visibility = ["//visibility:public"], + deps = [ + "//pkg/log:go_default_library", + "//pkg/private/serrors:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "export_test.go", + "multihomed_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//pkg/private/xtest:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@com_github_stretchr_testify//suite:go_default_library", + ], +) diff --git a/pkg/snet/multihomed/export_test.go b/pkg/snet/multihomed/export_test.go new file mode 100644 index 0000000000..d1f4290cff --- /dev/null +++ b/pkg/snet/multihomed/export_test.go @@ -0,0 +1,49 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multihomed + +import ( + "net/netip" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func MustGetEgressIpAddresses(t *testing.T) []netip.Addr { + addrs, err := egressIpAddresses() + require.NoError(t, err) + return addrs +} + +func GetInternalMutex() *sync.RWMutex { + return &muRemoteToEgress +} + +func StopTicker() { + stopContinuousCheckInterfaces() +} + +func GetRemoteToEgressMap() map[netip.Addr]netip.Addr { + return remoteToEgress +} + +func ReplaceRemoteToEgressMap(newMap map[netip.Addr]netip.Addr) { + remoteToEgress = newMap +} + +func GetEgressesLastState() *[]netip.Addr { + return &localAddresses +} diff --git a/pkg/snet/multihomed/interfaces.go b/pkg/snet/multihomed/interfaces.go new file mode 100644 index 0000000000..3344c4aba8 --- /dev/null +++ b/pkg/snet/multihomed/interfaces.go @@ -0,0 +1,159 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multihomed + +// The reasons to keep track of the current local addresses are that these two things can happen: +// 1. The interface is brought down and can't be used anymore. This requires the new +// packet to be sent using a different interface, if any is available. +// 2. The interface changed address, which needs us to update the tables and record the +// new address in use. +// The second event deals with the address only, without touching the routing table, while the +// first one modifies the routing table. For our purposes, both events modify the local address +// of the interface, and for both events the solution is to query the kernel again. +// This is why on the event of any address change, we completely clear the table, forcing +// the caller to perform a syscall to find the appropriate route. + +// XXX(juagargi): The right way to keep this routing information updated is to use netlink. +// We however just keep a cache of the last used remote addresses mapped to our interfaces' +// local addresses. Additionally, if the current interfaces' local addresses change, we +// completely clear the cache. + +import ( + "fmt" + "net" + "net/netip" + "os" + "slices" + "sort" + "sync" + "testing" + "time" + + "github.com/scionproto/scion/pkg/log" + "github.com/scionproto/scion/pkg/private/serrors" +) + +const ( + CheckInterfacesPeriod = time.Second + MaxAllowedCacheSize = 65536 // Maximum number of entries present in `remoteToEgress`. +) + +var ( + remoteToEgress map[netip.Addr]netip.Addr = make(map[netip.Addr]netip.Addr) + muRemoteToEgress sync.RWMutex = sync.RWMutex{} + localAddresses = make([]netip.Addr, 0) + stopTicker = make(chan struct{}) +) + +func init() { + go func() { + defer log.HandlePanic() + continuousCheckInterfaces() + }() +} + +// StopContinuousCheckInterfaces is used in tests where they need to stop the running +// goroutine that checks the state of the local interfaces. +func StopContinuousCheckInterfaces(*testing.T) { + if testing.Testing() { + stopContinuousCheckInterfaces() + } +} + +func continuousCheckInterfaces() { + clearCacheIfLocalChanges() + ticker := time.NewTicker(CheckInterfacesPeriod) +loop: + for { + select { + case <-ticker.C: + clearCacheIfLocalChanges() + case <-stopTicker: + ticker.Stop() + break loop + } + } +} + +func clearCacheIfLocalChanges() { + addrs := getInterfacesLocalAddresses() + if addrs == nil { + // Internal error, bail. + return + } + + // Compare with previous result. + if slices.Equal(addrs, localAddresses) { + // They are the same, bail. + return + } + + // Not equal, invalidate every entry. + invalidateAll() + // And store current state. + localAddresses = addrs +} + +func getInterfacesLocalAddresses() []netip.Addr { + // We only look at the local addresses. If they are not identical to the last call, + // remove all entries from the map, forcing the callers to obtain a new routed egress. + addrs, err := egressIpAddresses() + if err != nil { + // What do we do in this case? + // We should at least log the error and erase all entries in the table. + fmt.Fprintf(os.Stderr, "cannot list the network interfaces and their addresses: %s", err) + invalidateAll() + return nil + } + // Sort the result. + sort.Slice(addrs, func(i, j int) bool { + return addrs[i].Compare(addrs[j]) < 0 + }) + return addrs +} + +func invalidateAll() { + muRemoteToEgress.Lock() + remoteToEgress = make(map[netip.Addr]netip.Addr) + muRemoteToEgress.Unlock() +} + +func egressIpAddresses() ([]netip.Addr, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, serrors.Wrap("listing interfaces", err) + } + ipAddrs := make([]netip.Addr, 0, len(interfaces)) + + for _, iface := range interfaces { + addrs, err := iface.Addrs() + if err != nil { + return nil, serrors.Wrap("getting interface addresses", err, "interface", iface.Name) + } + for _, addr := range addrs { + ipAddr, ok := addr.(*net.IPNet) + if ok { + a, _ := netip.AddrFromSlice(ipAddr.IP) + ipAddrs = append(ipAddrs, a) + } + } + } + + return ipAddrs, nil +} + +func stopContinuousCheckInterfaces() { + stopTicker <- struct{}{} +} diff --git a/pkg/snet/multihomed/multihomed_test.go b/pkg/snet/multihomed/multihomed_test.go new file mode 100644 index 0000000000..3519a72c7d --- /dev/null +++ b/pkg/snet/multihomed/multihomed_test.go @@ -0,0 +1,194 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multihomed_test + +import ( + "crypto/rand" + "net/netip" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/scionproto/scion/pkg/private/xtest" + "github.com/scionproto/scion/pkg/snet/multihomed" +) + +func TestMultihomed(t *testing.T) { + suite.Run(t, NewMultihomedTestSuite()) +} + +// MultihomedTestSuite ensures that each test in this package is run correctly, even +// in the presence of test functions that alter the internal behaviour of mutexes or +// critical data structures. This is done by protecting the execution of each test function +// with a RWMutex, allowing "regular" tests to obtain a read lock, and "special" tests +// to get a write lock, forcing them run in isolation. +// It allows to call t.Parallel() in any and all test functions. +type MultihomedTestSuite struct { + suite.Suite + muInternalsIsolated sync.RWMutex +} + +func NewMultihomedTestSuite() *MultihomedTestSuite { + return &MultihomedTestSuite{} +} + +func (s *MultihomedTestSuite) SetupTest() { + s.muInternalsIsolated.RLock() +} + +func (s *MultihomedTestSuite) TearDownTest() { + s.muInternalsIsolated.RUnlock() +} + +func (s *MultihomedTestSuite) TestListInterfaces() { + t := s.T() + t.Parallel() + addrs := multihomed.MustGetEgressIpAddresses(t) + require.NotEmpty(t, addrs) +} + +func (s *MultihomedTestSuite) TestInternalEgressCache() { + t := s.T() + // We require this function to run in isolation: lock every other test. + s.muInternalsIsolated.RUnlock() + s.muInternalsIsolated.Lock() + defer func() { + // Because we hold the write lock, unlock it. + s.muInternalsIsolated.Unlock() + // Because the test suite will expect a read lock, get it. + s.muInternalsIsolated.RLock() + }() + + // Synchronize with the internal ticker routine to ensure it finished the update. + checkTicker := time.NewTicker(10 * time.Millisecond) + for i, _ := 0, <-checkTicker.C; i < 10; i, _ = i+1, <-checkTicker.C { + // Check that the egress table is not empty. + if len(*multihomed.GetEgressesLastState()) > 0 { + break + } + } + checkTicker.Stop() + require.NotEmpty(t, *multihomed.GetEgressesLastState()) + + // Stop internal refresh method. + multihomed.StopTicker() + + // Clear map. + multihomed.ReplaceRemoteToEgressMap(make(map[netip.Addr]netip.Addr)) + require.Empty(t, multihomed.GetRemoteToEgressMap()) + + // Create a pretend remote endpoint. + const mockRemoteAddress = "127.1.2.3" + const mockEgressAddress = "127.1.2.100" + mockRemote := xtest.MustParseUDPAddr(t, mockRemoteAddress+":22") + + // Add mock remote entry to map. + multihomed.ReplaceRemoteToEgressMap(map[netip.Addr]netip.Addr{ + netip.MustParseAddr(mockRemoteAddress): netip.MustParseAddr(mockEgressAddress), + }) + + // Actual test, get the egress address for the remote. + expected := xtest.MustParseIP(t, mockEgressAddress).To4() + got, err := multihomed.OutboundIP(mockRemote) + require.NoError(t, err) + require.Equal(t, expected, got) +} + +// BenchmarkSyncMapWrites (and the other 3 analogous benchmarks) are used to check the performance +// of a sync.Map (any->any) and a regular map (IP->IP) with a RWMutex. +func BenchmarkSyncMapWrites(b *testing.B) { + // Create a set of `size` IP addresses. + addrs := generateIpAddrs(b.N) + + m := sync.Map{} + b.ResetTimer() + storeInSyncMap(&m, addrs) +} + +func BenchmarkSyncMapReads(b *testing.B) { + addrs := generateIpAddrs(b.N) + m := sync.Map{} + storeInSyncMap(&m, addrs) + + // Refrain optimizer from removing code by adding the values to a discard buffer. + discardBuff := make([]netip.Addr, b.N) + b.ResetTimer() + for i, addr := range addrs { + a, ok := m.Load(addr) + addr = a.(netip.Addr) + _ = ok + discardBuff[i] = addr + } + b.StopTimer() + require.NotEmpty(b, discardBuff) + require.Len(b, discardBuff, b.N) +} + +func BenchmarkMuMapWrites(b *testing.B) { + addrs := generateIpAddrs(b.N) + + m := make(map[netip.Addr]netip.Addr) + mu := sync.RWMutex{} + b.ResetTimer() + storeInMuMap(m, &mu, addrs) +} + +func BenchmarkMuMapReads(b *testing.B) { + addrs := generateIpAddrs(b.N) + m := make(map[netip.Addr]netip.Addr) + mu := sync.RWMutex{} + storeInMuMap(m, &mu, addrs) + + // Refrain optimizer from removing code by adding the values to a discard buffer. + discardBuff := make([]netip.Addr, b.N) + b.ResetTimer() + for i, addr := range addrs { + mu.RLock() + addr, ok := m[addr] + mu.RUnlock() + _ = ok + discardBuff[i] = addr + } + b.StopTimer() + require.NotEmpty(b, discardBuff) + require.Len(b, discardBuff, b.N) +} + +func generateIpAddrs(size int) []netip.Addr { + addrs := make([]netip.Addr, size) + raw := [4]byte{} + for i := range size { + rand.Read(raw[:]) + addrs[i] = netip.AddrFrom4(raw) + } + return addrs +} + +func storeInSyncMap(m *sync.Map, addrs []netip.Addr) { + for _, addr := range addrs { + m.Store(addr, addr) + } +} + +func storeInMuMap(m map[netip.Addr]netip.Addr, mu *sync.RWMutex, addrs []netip.Addr) { + for _, addr := range addrs { + mu.Lock() + m[addr] = addr + mu.Unlock() + } +} diff --git a/pkg/snet/multihomed/outbound_addr.go b/pkg/snet/multihomed/outbound_addr.go new file mode 100644 index 0000000000..88bb6c10eb --- /dev/null +++ b/pkg/snet/multihomed/outbound_addr.go @@ -0,0 +1,75 @@ +// Copyright 2025 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multihomed + +import ( + "net" + "net/netip" + + "github.com/scionproto/scion/pkg/private/serrors" +) + +// OutboundIP returns the IP address used by this host to dial to the specified remote host. +// The port value in the remote udp address is irrelevant. +// It relies on a previously populated table that maps remote addresses to egress addresses. +// If the remote is not present, it is added. +// Note that NAT address discovery support in scionproto via STUN will also dial periodically +// connections to the STUN server, which should be enough to find the local address used +// to route to the next hop. +func OutboundIP(nextHop *net.UDPAddr) (net.IP, error) { + remote, ok := netip.AddrFromSlice(nextHop.IP) + if !ok { + return nil, serrors.New("invalid IP address", "address", nextHop.IP) + } + + // Check if the table contains an entry. + muRemoteToEgress.RLock() + egress, ok := remoteToEgress[remote] + muRemoteToEgress.RUnlock() + if ok { + return net.IP(egress.AsSlice()), nil + } + + // Not found, find it and add it. The dialing involves a syscall, but no network traffic. + eg, err := dialRemote(nextHop) + if err != nil { + return nil, err + } + egress, _ = netip.AddrFromSlice(eg) + + muRemoteToEgress.Lock() + // Check if our cache is not too big already. + if len(remoteToEgress) < MaxAllowedCacheSize { + remoteToEgress[remote] = egress + } + muRemoteToEgress.Unlock() + + return eg, nil +} + +// dialRemote creates a socket used to send UDP packets to the remote endpoint. +// Note that while a syscall is performed (two including Close), there will be no network traffic. +// Anyhow, this is somewhat expensive, so try to reduce its usage. +func dialRemote(raddr *net.UDPAddr) (net.IP, error) { + conn, err := net.DialUDP("udp", nil, raddr) + if err != nil { + return nil, err + } + defer conn.Close() + + // The conn object is always a net.UDPConn, with LocalAddr statically returning + // always a *net.UDPAddr. + return conn.LocalAddr().(*net.UDPAddr).IP, nil +} diff --git a/pkg/snet/reader.go b/pkg/snet/reader.go index 88f8e6ef2d..715de4803b 100644 --- a/pkg/snet/reader.go +++ b/pkg/snet/reader.go @@ -16,7 +16,6 @@ package snet import ( "net" - "net/netip" "sync" "time" @@ -91,37 +90,6 @@ func (c *scionConnReader) read(b []byte) (int, *UDPAddr, error) { return 0, nil, serrors.New("unexpected payload", "type", common.TypeOf(pkt.Payload)) } - pktAddrPort := netip.AddrPortFrom(pkt.Destination.Host.IP(), udp.DstPort) - if c.local.IA != pkt.Destination.IA { - return 0, nil, serrors.New("packet is destined to a different IA", - "local_isd_as", c.local.IA, - "local_host", c.local.Host, - "pkt_destination_isd_as", pkt.Destination.IA, - "pkt_destination_host", pktAddrPort, - ) - } - - // XXX(JordiSubira): We explicitly forbid nil or unspecified address in the current constructor - // for Conn. - // If this were ever to change, we would always fall into the following if statement, then - // we would like to replace this logic (e.g., using IP_PKTINFO, with its caveats). - if c.local.Host.AddrPort() != pktAddrPort { - - // If the client is behind a NAT, the SCION packet will hold the mapped external address, - // which is expected to be different from the local address. To handle this case, we check - // whether the underlying connection is a stunConn, which indicates that NAT traversal - // is in use. - // TODO: Is it necessary to check that the address matches one of the mapped addresses? - if !c.hasSTUN { - return 0, nil, serrors.New("packet is destined to a different host", - "local_isd_as", c.local.IA, - "local_host", c.local.Host, - "pkt_destination_isd_as", pkt.Destination.IA, - "pkt_destination_host", pktAddrPort, - ) - } - } - // Extract remote address. // Copy the address data to prevent races. See // https://github.com/scionproto/scion/issues/1659. diff --git a/pkg/snet/snet.go b/pkg/snet/snet.go index 386d946096..f747c04435 100644 --- a/pkg/snet/snet.go +++ b/pkg/snet/snet.go @@ -95,17 +95,13 @@ type SCIONNetwork struct { } // OpenRaw returns a PacketConn which listens on the specified address. -// Nil or unspecified addresses are not supported. // If the address port is 0 a valid and free SCION/UDP port is automatically chosen. // Otherwise, the specified port must be a valid SCION/UDP port. func (n *SCIONNetwork) OpenRaw(ctx context.Context, addr *net.UDPAddr) (PacketConn, error) { var pconn *net.UDPConn var err error - if addr == nil || addr.IP.IsUnspecified() { - return nil, serrors.New("nil or unspecified address is not supported") - } start, end := n.Topology.PortRange.Start, n.Topology.PortRange.End - if addr.Port == 0 { + if addr == nil || addr.Port == 0 { pconn, err = listenUDPRange(addr, start, end) } else { if addr.Port < int(start) || addr.Port > int(end) { @@ -199,7 +195,9 @@ func (n *SCIONNetwork) Listen( return NewCookedConn(packetConn, n.Topology, WithReplyPather(n.ReplyPather)) } -func listenUDPRange(addr *net.UDPAddr, start, end uint16) (*net.UDPConn, error) { +// listenUDPRange creates a new net.UDPConn using a suitable port between the specified range. +// If laddr is not nil, the returned connection is bound to its IP address. +func listenUDPRange(laddr *net.UDPAddr, start, end uint16) (*net.UDPConn, error) { // XXX(JordiSubira): For now, we iterate on the complete SCION/UDP // range, in decreasing order, taking the first unused port. // @@ -220,11 +218,18 @@ func listenUDPRange(addr *net.UDPAddr, start, end uint16) (*net.UDPConn, error) if start < 1024 { restrictedStart = 1024 } + + // Local address used later in a loop to check if the different ports are available. + tryLocalAddr := &net.UDPAddr{} + if laddr != nil { + tryLocalAddr.IP = laddr.IP + tryLocalAddr.Zone = laddr.Zone + } + for port := end; port >= restrictedStart; port-- { - pconn, err := net.ListenUDP(addr.Network(), &net.UDPAddr{ - IP: addr.IP, - Port: int(port), - }) + tryLocalAddr.Port = int(port) + pconn, err := net.ListenUDP("udp", tryLocalAddr) + if err == nil { return pconn, nil } diff --git a/pkg/snet/writer.go b/pkg/snet/writer.go index dcf6d5d11a..301becef85 100644 --- a/pkg/snet/writer.go +++ b/pkg/snet/writer.go @@ -24,6 +24,7 @@ import ( "github.com/scionproto/scion/pkg/addr" "github.com/scionproto/scion/pkg/private/serrors" + "github.com/scionproto/scion/pkg/snet/multihomed" "github.com/scionproto/scion/private/topology" ) @@ -85,6 +86,7 @@ func (c *scionConnWriter) WriteTo(b []byte, raddr net.Addr) (int, error) { if !ok { return 0, serrors.New("invalid listen host IP", "ip", c.local.Host.IP) } + listenHostPort := uint16(c.local.Host.Port) // Rewrite source address if STUN is in use @@ -99,6 +101,17 @@ func (c *scionConnWriter) WriteTo(b []byte, raddr net.Addr) (int, error) { return 0, err } + if listenHostIP.IsUnspecified() { + // Sending data to an unbound socket, we need to find the appropriate local address + // to write it as host address in the SCION packet. The interface to use in the local + // host will be that one routed to reach the next hop. + localIP, err := multihomed.OutboundIP(nextHop) + if err != nil { + return 0, serrors.Wrap("interface IP not bound and cannot find one", err) + } + listenHostIP, _ = netip.AddrFromSlice(localIP) + } + pkt := &Packet{ Bytes: Bytes(c.buffer), PacketInfo: PacketInfo{