diff --git a/proxy/socks/protocol.go b/proxy/socks/protocol.go
index bf4f61a5..5d4cfd9d 100644
--- a/proxy/socks/protocol.go
+++ b/proxy/socks/protocol.go
@@ -209,11 +209,12 @@ func (s *ServerSession) handshake5(nMethod byte, reader io.Reader, writer net.Co
 		}
 		responsePort = net.Port(udpHub.LocalAddr().(*net.UDPAddr).Port)
 		expectedRemote := &gonet.UDPAddr{}
-		if request.Address.IP().IsUnspecified() {
+		// UDP Associate should not specify a domain as source IP
+		if request.Address.Family().IsDomain() || request.Address.IP().IsUnspecified() {
 			expectedRemote.IP = writer.RemoteAddr().(*net.TCPAddr).IP // unix?
 		} else {
-			expectedRemote.IP = request.Address.IP() // panic?
-			expectedRemote.Port = int(request.Port)  // 0 is allowed
+			expectedRemote.IP = request.Address.IP()
+			expectedRemote.Port = int(request.Port) // 0 is allowed
 		}
 		tempUDPConn = NewTempUDPConn(udpHub, writer, expectedRemote)
 	}
diff --git a/proxy/socks/server.go b/proxy/socks/server.go
index 53049dfe..e1d34a60 100644
--- a/proxy/socks/server.go
+++ b/proxy/socks/server.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	goerrors "errors"
 	"io"
+	"sync"
 	"time"
 
 	"github.com/xtls/xray-core/common"
@@ -216,18 +217,27 @@ func (s *Server) handleUDPPayload(ctx context.Context, conn stat.Connection, dis
 	defer udpServer.RemoveRay()
 
 	inbound := session.InboundFromContext(ctx)
-	if inbound != nil && inbound.Source.IsValid() {
-		errors.LogInfo(ctx, "client UDP connection from ", inbound.Source)
-	}
 
 	var dest *net.Destination
 
 	reader := buf.NewPacketReader(conn)
+	var changeRemote sync.Once
 	for {
 		mpayload, err := reader.ReadMultiBuffer()
 		if err != nil {
 			return err
 		}
+		changeRemote.Do(func() {
+			if inbound != nil {
+				newInbound := *inbound
+				// change source to real remote UDP address
+				newInbound.Source = net.DestinationFromAddr(conn.RemoteAddr())
+				newInbound.Local = net.DestinationFromAddr(conn.LocalAddr())
+				inbound = &newInbound
+				ctx = session.ContextWithInbound(ctx, inbound)
+				errors.LogInfo(ctx, "client UDP connection from ", inbound.Source)
+			}
+		})
 
 		for _, payload := range mpayload {
 			request, err := DecodeUDPPacket(payload)